{
  "_id": "6a06d86a7757ba2767e24222",
  "Package": "robotstxt",
  "Type": "Package",
  "Title": "A 'robots.txt' Parser and 'Webbot'/'Spider'/'Crawler'\nPermissions Checker",
  "Version": "0.7.15.9000",
  "Authors@R": "c(\nperson(\n\"Pedro\", \"Baltazar\", role = c(\"ctb\"),\nemail = \"pedrobtz@gmail.com\"\n),\nperson(\n\"Jordan\", \"Bradford\", role = c(\"cre\"),\nemail = \"jrdnbradford@gmail.com\"\n),\nperson(\n\"Peter\", \"Meissner\", role = c(\"aut\"),\nemail = \"retep.meissner@gmail.com\"\n),\nperson(\n\"Kun\", \"Ren\", email = \"mail@renkun.me\", role = c(\"aut\", \"cph\"),\ncomment = \"Author and copyright holder of list_merge.R.\"\n),\nperson(\"Oliver\", \"Keys\", role = \"ctb\", comment = \"original release code review\"),\nperson(\"Rich\", \"Fitz John\", role = \"ctb\", comment = \"original release code review\")\n)",
  "Description": "Provides functions to download and parse 'robots.txt'\nfiles. Ultimately the package makes it easy to check if bots\n(spiders, crawler, scrapers, ...) are allowed to access\nspecific resources on a domain.",
  "License": "MIT + file LICENSE",
  "BugReports": "https://github.com/ropensci/robotstxt/issues",
  "URL": "https://docs.ropensci.org/robotstxt/,\nhttps://github.com/ropensci/robotstxt",
  "VignetteBuilder": "knitr",
  "RoxygenNote": "7.3.2",
  "Encoding": "UTF-8",
  "Config/testthat/edition": "3",
  "Config/pak/sysreqs": "libicu-dev libssl-dev",
  "Repository": "https://ropensci.r-universe.dev",
  "Date/Publication": "2024-11-15 19:32:37 UTC",
  "RemoteUrl": "https://github.com/ropensci/robotstxt",
  "RemoteRef": "main",
  "RemoteSha": "d3d0a4d525e87204010d10162b01a921b05a6149",
  "NeedsCompilation": "no",
  "Packaged": {
    "Date": "2026-05-15 08:16:17 UTC",
    "User": "root"
  },
  "Author": "Pedro Baltazar [ctb],\nJordan Bradford [cre],\nPeter Meissner [aut],\nKun Ren [aut, cph] (Author and copyright holder of list_merge.R.),\nOliver Keys [ctb] (original release code review),\nRich Fitz John [ctb] (original release code review)",
  "Maintainer": "Jordan Bradford <jrdnbradford@gmail.com>",
  "MD5sum": "7b56d7415f539e07bbdac773cc767716",
  "_user": "ropensci",
  "_type": "src",
  "_file": "robotstxt_0.7.15.9000.tar.gz",
  "_fileid": "c3db172469051b0a2bd8357a173c4949386202dfa1dd33918a3c64e29e4b1148",
  "_filesize": 271987,
  "_sha256": "c3db172469051b0a2bd8357a173c4949386202dfa1dd33918a3c64e29e4b1148",
  "_created": "2026-05-15T08:16:17.000Z",
  "_published": "2026-05-15T08:25:14.292Z",
  "_distro": "noble",
  "_jobs": [
    {
      "job": 76144777829,
      "time": 117,
      "config": "linux-devel-x86_64",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "7012927335"
    },
    {
      "job": 76144777803,
      "time": 135,
      "config": "linux-release-x86_64",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7012930852"
    },
    {
      "job": 76144777775,
      "time": 91,
      "config": "macos-oldrel-arm64",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "7012921158"
    },
    {
      "job": 76144777758,
      "time": 80,
      "config": "macos-release-arm64",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7012919014"
    },
    {
      "job": 76144069073,
      "time": 155,
      "config": "pkgdown",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7012890227"
    },
    {
      "job": 76144069072,
      "time": 216,
      "config": "source",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7012901712"
    },
    {
      "job": 76144777850,
      "time": 108,
      "config": "wasm-release",
      "r": "4.5.1",
      "check": "OK",
      "artifact": "7012925008"
    },
    {
      "job": 76144777787,
      "time": 80,
      "config": "windows-devel",
      "r": "4.7.0",
      "check": "OK",
      "artifact": "7012919395"
    },
    {
      "job": 76144777799,
      "time": 82,
      "config": "windows-oldrel",
      "r": "4.5.3",
      "check": "OK",
      "artifact": "7012919500"
    },
    {
      "job": 76144777808,
      "time": 78,
      "config": "windows-release",
      "r": "4.6.0",
      "check": "OK",
      "artifact": "7012918866"
    }
  ],
  "_buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713",
  "_status": "success",
  "_host": "GitHub-Actions",
  "_upstream": "https://github.com/ropensci/robotstxt",
  "_commit": {
    "id": "d3d0a4d525e87204010d10162b01a921b05a6149",
    "author": "Jordan Bradford <36420801+jrdnbradford@users.noreply.github.com>",
    "committer": "GitHub <noreply@github.com>",
    "message": "Merge pull request #94 from ropensci/simplify-get_robotstxt_http_get\n\nRewrite `get_robotstxt_http_get`",
    "time": 1731699157
  },
  "_maintainer": {
    "name": "Jordan Bradford",
    "email": "jrdnbradford@gmail.com",
    "login": "jrdnbradford",
    "uuid": 36420801
  },
  "_registered": true,
  "_dependencies": [
    {
      "package": "R",
      "version": ">= 3.0.0",
      "role": "Depends"
    },
    {
      "package": "stringr",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "httr",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "spiderbar",
      "version": ">= 0.2.0",
      "role": "Imports"
    },
    {
      "package": "future.apply",
      "version": ">= 1.0.0",
      "role": "Imports"
    },
    {
      "package": "magrittr",
      "role": "Imports"
    },
    {
      "package": "utils",
      "role": "Imports"
    },
    {
      "package": "knitr",
      "role": "Suggests"
    },
    {
      "package": "rmarkdown",
      "role": "Suggests"
    },
    {
      "package": "dplyr",
      "role": "Suggests"
    },
    {
      "package": "testthat",
      "version": ">= 3.0.0",
      "role": "Suggests"
    },
    {
      "package": "covr",
      "role": "Suggests"
    },
    {
      "package": "curl",
      "role": "Suggests"
    }
  ],
  "_owner": "ropensci",
  "_selfowned": true,
  "_usedby": 5,
  "_updates": [],
  "_tags": [],
  "_topics": [
    "crawler",
    "peer-reviewed",
    "robotstxt",
    "scraper",
    "spider",
    "webscraping"
  ],
  "_stars": 69,
  "_contributors": [
    {
      "user": "petermeissner",
      "count": 248,
      "uuid": 939139
    },
    {
      "user": "jrdnbradford",
      "count": 32,
      "uuid": 36420801
    },
    {
      "user": "pedrobtz",
      "count": 19,
      "uuid": 2864346
    },
    {
      "user": "dmi3kno",
      "count": 5,
      "uuid": 13419011
    },
    {
      "user": "gittaca",
      "count": 3,
      "uuid": 17674964
    },
    {
      "user": "maelle",
      "count": 2,
      "uuid": 8360597
    },
    {
      "user": "sckott",
      "count": 2,
      "uuid": 577668
    },
    {
      "user": "karthik",
      "count": 1,
      "uuid": 138494
    },
    {
      "user": "mine-cetinkaya-rundel",
      "count": 1,
      "uuid": 5965649
    }
  ],
  "_userbio": {
    "uuid": 1200269,
    "type": "organization",
    "name": "rOpenSci",
    "description": "Tools and R Packages for Open Science"
  },
  "_downloads": {
    "count": 1576,
    "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/robotstxt"
  },
  "_devurl": "https://github.com/ropensci/robotstxt",
  "_pkgdown": "https://docs.ropensci.org/robotstxt/",
  "_searchresults": 422,
  "_metadata": {
    "review": {
      "id": 25,
      "status": "reviewed",
      "version": "0.1.0",
      "organization": "rOpenSci Software Review",
      "url": "https://github.com/ropensci/software-review/issues/25"
    },
    "ropensci_category": "http-tools"
  },
  "_rbuild": "4.6.0",
  "_assets": [
    "extra/citation.cff",
    "extra/citation.html",
    "extra/citation.json",
    "extra/citation.txt",
    "extra/contents.json",
    "extra/NEWS.html",
    "extra/NEWS.txt",
    "extra/readme.html",
    "extra/readme.md",
    "extra/robotstxt.html",
    "manual.pdf"
  ],
  "_homeurl": "https://github.com/ropensci/robotstxt",
  "_realowner": "ropensci",
  "_cranurl": true,
  "_releases": [
    {
      "version": "0.1.2",
      "date": "2016-02-08"
    },
    {
      "version": "0.3.2",
      "date": "2016-04-27"
    },
    {
      "version": "0.4.0",
      "date": "2017-07-16"
    },
    {
      "version": "0.4.1",
      "date": "2017-09-01"
    },
    {
      "version": "0.5.2",
      "date": "2017-11-12"
    },
    {
      "version": "0.6.0",
      "date": "2018-02-11"
    },
    {
      "version": "0.6.2",
      "date": "2018-07-18"
    },
    {
      "version": "0.7.4",
      "date": "2020-05-31"
    },
    {
      "version": "0.7.7",
      "date": "2020-06-27"
    },
    {
      "version": "0.7.8",
      "date": "2020-07-25"
    },
    {
      "version": "0.7.13",
      "date": "2020-09-03"
    },
    {
      "version": "0.7.15",
      "date": "2024-08-29"
    }
  ],
  "_exports": [
    "%>%",
    "get_robotstxt",
    "get_robotstxt_http_get",
    "get_robotstxts",
    "is_valid_robotstxt",
    "on_client_error_default",
    "on_domain_change_default",
    "on_file_type_mismatch_default",
    "on_not_found_default",
    "on_redirect_default",
    "on_server_error_default",
    "on_sub_domain_change_default",
    "on_suspect_content_default",
    "parse_robotstxt",
    "paths_allowed",
    "request_handler_handler",
    "robotstxt",
    "rt_last_http",
    "rt_request_handler"
  ],
  "_help": [
    {
      "page": "pipe",
      "title": "re-export magrittr pipe operator",
      "topics": [
        "%>%"
      ]
    },
    {
      "page": "as.list.robotstxt_text",
      "title": "Convert robotstxt_text to list",
      "topics": [
        "as.list.robotstxt_text"
      ]
    },
    {
      "page": "fix_url",
      "title": "Add http protocal if missing from URL",
      "topics": [
        "fix_url"
      ]
    },
    {
      "page": "get_robotstxt",
      "title": "Download a robots.txt file",
      "topics": [
        "get_robotstxt"
      ]
    },
    {
      "page": "get_robotstxts",
      "title": "Download multiple robotstxt files",
      "topics": [
        "get_robotstxts"
      ]
    },
    {
      "page": "guess_domain",
      "title": "Guess a domain from path",
      "topics": [
        "guess_domain"
      ]
    },
    {
      "page": "http_domain_changed",
      "title": "Check if HTTP domain changed",
      "topics": [
        "http_domain_changed"
      ]
    },
    {
      "page": "http_subdomain_changed",
      "title": "Check if HTTP subdomain changed",
      "topics": [
        "http_subdomain_changed"
      ]
    },
    {
      "page": "http_was_redirected",
      "title": "Check if HTTP redirect occurred",
      "topics": [
        "http_was_redirected"
      ]
    },
    {
      "page": "is_suspect_robotstxt",
      "title": "Check if file is valid / parsable robots.txt file",
      "topics": [
        "is_suspect_robotstxt"
      ]
    },
    {
      "page": "is_valid_robotstxt",
      "title": "Validate if a file is valid / parsable robots.txt file",
      "topics": [
        "is_valid_robotstxt"
      ]
    },
    {
      "page": "list_merge",
      "title": "Merge a number of named lists in sequential order",
      "topics": [
        "list_merge"
      ]
    },
    {
      "page": "null_to_default",
      "title": "Return default value if NULL",
      "topics": [
        "null_to_default"
      ]
    },
    {
      "page": "parse_robotstxt",
      "title": "Parse a robots.txt file",
      "topics": [
        "parse_robotstxt"
      ]
    },
    {
      "page": "paths_allowed",
      "title": "Check if a bot has permissions to access page(s)",
      "topics": [
        "paths_allowed"
      ]
    },
    {
      "page": "paths_allowed_worker_spiderbar",
      "title": "Check if a spiderbar bot has permissions to access page(s)",
      "topics": [
        "paths_allowed_worker_spiderbar"
      ]
    },
    {
      "page": "print.robotstxt",
      "title": "Print robotstxt",
      "topics": [
        "print.robotstxt"
      ]
    },
    {
      "page": "print.robotstxt_text",
      "title": "Print robotstxt's text",
      "topics": [
        "print.robotstxt_text"
      ]
    },
    {
      "page": "remove_domain",
      "title": "Remove domain from path",
      "topics": [
        "remove_domain"
      ]
    },
    {
      "page": "request_handler_handler",
      "title": "Handle robotstxt handlers",
      "topics": [
        "request_handler_handler"
      ]
    },
    {
      "page": "robotstxt",
      "title": "Generate a representation of a robots.txt file",
      "topics": [
        "robotstxt"
      ]
    },
    {
      "page": "rt_cache",
      "title": "Get the robotstxt cache",
      "topics": [
        "rt_cache"
      ]
    },
    {
      "page": "get_robotstxt_http_get",
      "title": "Storage for HTTP request response objects",
      "topics": [
        "get_robotstxt_http_get",
        "rt_last_http"
      ]
    },
    {
      "page": "rt_request_handler",
      "title": "Handle robotstxt object retrieved from HTTP request",
      "topics": [
        "on_client_error_default",
        "on_domain_change_default",
        "on_file_type_mismatch_default",
        "on_not_found_default",
        "on_redirect_default",
        "on_server_error_default",
        "on_sub_domain_change_default",
        "on_suspect_content_default",
        "rt_request_handler"
      ]
    }
  ],
  "_readme": "https://github.com/ropensci/robotstxt/raw/main/README.md",
  "_rundeps": [
    "askpass",
    "cli",
    "codetools",
    "curl",
    "digest",
    "future",
    "future.apply",
    "globals",
    "glue",
    "httr",
    "jsonlite",
    "lifecycle",
    "listenv",
    "magrittr",
    "mime",
    "openssl",
    "parallelly",
    "R6",
    "Rcpp",
    "rlang",
    "spiderbar",
    "stringi",
    "stringr",
    "sys",
    "vctrs"
  ],
  "_vignettes": [
    {
      "source": "using_robotstxt.Rmd",
      "filename": "using_robotstxt.html",
      "title": "Using Robotstxt",
      "author": "Peter Meissner",
      "engine": "knitr::rmarkdown",
      "headings": [
        "Description",
        "Robots.txt files",
        "Fast food usage for the uninterested",
        "Example Usage",
        "object oriented style",
        "functional style"
      ],
      "created": "2016-01-09 20:19:32",
      "modified": "2024-08-24 11:05:33",
      "commits": 9
    }
  ],
  "_score": 9.43986900523611,
  "_indexed": true,
  "_nocasepkg": "robotstxt",
  "_universes": [
    "ropensci",
    "jrdnbradford"
  ],
  "_binaries": [
    {
      "r": "4.7.0",
      "os": "linux",
      "version": "0.7.15.9000",
      "date": "2026-05-15T08:18:17.000Z",
      "distro": "noble",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "925effbce9bfddcc0e855d2eb03a366a2aba93eadaacdc5d81e3634a77999524",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    },
    {
      "r": "4.6.0",
      "os": "linux",
      "version": "0.7.15.9000",
      "date": "2026-05-15T08:18:34.000Z",
      "distro": "noble",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "70048e56e07a3b220b57a0c02cf91c994b851a2603f3c893fda35292e5412d45",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    },
    {
      "r": "4.5.3",
      "os": "mac",
      "version": "0.7.15.9000",
      "date": "2026-05-15T08:17:54.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "ba416727db4555c231345e14e45de3ef5913a99ebaa342f019587e335c9e576c",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    },
    {
      "r": "4.6.0",
      "os": "mac",
      "version": "0.7.15.9000",
      "date": "2026-05-15T08:17:45.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "286d9df72ff22a1ee417b546e25bea24332bbe7e8ad419348ce77b10289eede7",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    },
    {
      "r": "4.7.0",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-05-15T08:17:34.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "cad54ecdbb4108a3102fd978d6ca669390d138fa97892c69803bb8ba925d3cb0",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    },
    {
      "r": "4.5.3",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-05-15T08:17:33.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "b9874ffdd0fb5ed318ce537f0fa68e50e6624d14a6a75a164ba710d4ee012b33",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    },
    {
      "r": "4.6.0",
      "os": "win",
      "version": "0.7.15.9000",
      "date": "2026-05-15T08:17:30.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "4f9d12b4aac35c27c37300c4a543e1636d5b680b2a198d7d4ef37c94390b6b9b",
      "status": "success",
      "check": "OK",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    },
    {
      "r": "4.6.0",
      "os": "wasm",
      "version": "0.7.15.9000",
      "date": "2026-05-22T07:43:37.000Z",
      "commit": "d3d0a4d525e87204010d10162b01a921b05a6149",
      "fileid": "1cf8dd1deed5b289a88b4aac3d30158421c07a4f11408e07e437558d258d97bd",
      "status": "success",
      "buildurl": "https://github.com/r-universe/ropensci/actions/runs/25907560713"
    }
  ]
}