{ "_id": "6a6ddf604422390e5d865325", "Package": "tokenizers", "Type": "Package", "Title": "Fast, Consistent Tokenization of Natural Language Text", "Version": "0.3.1", "Date": "2024-03-27", "Description": "Convert natural language text into tokens. Includes\ntokenizers for shingled n-grams, skip n-grams, words, word\nstems, sentences, paragraphs, characters, shingled characters,\nlines, Penn Treebank, regular expressions, as well as functions\nfor counting characters, words, and sentences, and a function\nfor splitting longer texts into separate documents, each with\nthe same number of words. The tokenizers have a consistent\ninterface, and the package is built on the 'stringi' and 'Rcpp'\npackages for fast yet correct tokenization in 'UTF-8'.", "License": "MIT + file LICENSE", "LazyData": "yes", "Authors@R": "c(person(\"Thomas\", \"Charlon\", role = c(\"aut\", \"cre\"),\nemail = \"charlon@protonmail.com\",\ncomment = c(ORCID = \"0000-0001-7497-0470\")),\nperson(\"Lincoln\", \"Mullen\", role = c(\"aut\"),\nemail = \"lincoln@lincolnmullen.com\",\ncomment = c(ORCID = \"0000-0001-5103-6917\")),\nperson(\"Os\", \"Keyes\", role = c(\"ctb\"),\nemail = \"ironholds@gmail.com\",\ncomment = c(ORCID = \"0000-0001-5196-609X\")),\nperson(\"Dmitriy\", \"Selivanov\", role = c(\"ctb\"),\nemail = \"selivanov.dmitriy@gmail.com\"),\nperson(\"Jeffrey\", \"Arnold\", role = c(\"ctb\"),\nemail = \"jeffrey.arnold@gmail.com\",\ncomment = c(ORCID = \"0000-0001-9953-3904\")),\nperson(\"Kenneth\", \"Benoit\", role = c(\"ctb\"),\nemail = \"kbenoit@lse.ac.uk\",\ncomment = c(ORCID = \"0000-0002-0797-564X\")))", "URL": "https://docs.ropensci.org/tokenizers/,\nhttps://github.com/ropensci/tokenizers", "BugReports": "https://github.com/ropensci/tokenizers/issues", "RoxygenNote": "7.3.1", "Encoding": "UTF-8", "VignetteBuilder": "knitr", "Config/pak/sysreqs": "libicu-dev", "Repository": "https://ropensci.r-universe.dev", "Date/Publication": "2024-03-27 09:33:34 UTC", "RemoteUrl": "https://github.com/ropensci/tokenizers", "RemoteRef": "master", "RemoteSha": "b80863d088d4b39695b602ca11e061ac34770ec7", "NeedsCompilation": "yes", "Packaged": { "Date": "2026-08-01 11:40:28 UTC", "User": "root" }, "Author": "Thomas Charlon [aut, cre] (ORCID:\n),\nLincoln Mullen [aut] (ORCID: ),\nOs Keyes [ctb] (ORCID: ),\nDmitriy Selivanov [ctb],\nJeffrey Arnold [ctb] (ORCID: ),\nKenneth Benoit [ctb] (ORCID: )", "Maintainer": "Thomas Charlon ", "_user": "ropensci", "_type": "src", "_file": "tokenizers_0.3.1.tar.gz", "_fileid": "https://r2.ropensci.org/179aaa5523e60e98994fa1b68c09a1ee089a26e5b4c051ad1a268930e23521e3", "_filesize": 576523, "_sha256": "179aaa5523e60e98994fa1b68c09a1ee089a26e5b4c051ad1a268930e23521e3", "_expires": "2026-11-09T11:58:22.000Z", "_created": "2026-08-01T11:40:28.000Z", "_published": "2026-08-01T11:58:24.658Z", "_jobs": [ { "job": 91364470519, "time": 118, "config": "linux-devel-arm64", "r": "4.7.0", "check": "NOTE", "artifact": "8817957614" }, { "job": 91364470522, "time": 154, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "NOTE", "artifact": "8817963360" }, { "job": 91364470525, "time": 120, "config": "linux-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "8817957654" }, { "job": 91364470520, "time": 142, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "8817961019" }, { "job": 91364470508, "time": 73, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "8818075322" }, { "job": 91364470518, "time": 267, "config": "macos-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "8818104763" }, { "job": 91364470505, "time": 102, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "8818082374" }, { "job": 91364470515, "time": 195, "config": "macos-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "8818101812" }, { "job": 91364163160, "time": 144, "config": "pkgdown", "r": "4.6.0", "check": "OK", "artifact": "8817920301" }, { "job": 91364163162, "time": 214, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "8817931296" }, { "job": 91364470496, "time": 120, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "8817957924" }, { "job": 91364470541, "time": 185, "config": "windows-devel-arm64", "r": "4.7.0", "check": "NOTE", "artifact": "8817969340" }, { "job": 91364470572, "time": 115, "config": "windows-devel-x86_64", "r": "4.7.0", "check": "NOTE", "artifact": "8817956882" }, { "job": 91364470552, "time": 106, "config": "windows-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "8817955897" }, { "job": 91364470554, "time": 221, "config": "windows-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "8817975768" }, { "job": 91364470558, "time": 94, "config": "windows-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "8817954199" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552", "_status": "success", "_upstream": "https://github.com/ropensci/tokenizers", "_commit": { "id": "b80863d088d4b39695b602ca11e061ac34770ec7", "author": "tcharlon ", "committer": "tcharlon ", "message": "new maintainer\n", "time": 1711532014 }, "_maintainer": { "name": "Thomas Charlon", "email": "charlon@protonmail.com", "login": "thomaschln", "linkedin": "in/thomas-charlon-meng-phd-aba0a3275", "orcid": "0000-0001-7497-0470", "description": "Harvard Medical School Researcher\nBiomedical Informatics @hms-dbmi\nCELEHS laboratory @celehs", "uuid": 2394508 }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 3.1.3", "role": "Depends" }, { "package": "Rcpp", "role": "LinkingTo" }, { "package": "stringi", "version": ">= 1.0.1", "role": "Imports" }, { "package": "Rcpp", "version": ">= 0.12.3", "role": "Imports" }, { "package": "SnowballC", "version": ">= 0.5.1", "role": "Imports" }, { "package": "covr", "role": "Suggests" }, { "package": "knitr", "role": "Suggests" }, { "package": "rmarkdown", "role": "Suggests" }, { "package": "stopwords", "version": ">= 0.9.0", "role": "Suggests" }, { "package": "testthat", "role": "Suggests" } ], "_owner": "ropensci", "_selfowned": true, "_usedby": 79, "_updates": [], "_tags": [], "_topics": [ "nlp", "peer-reviewed", "text-mining", "tokenizer", "cpp" ], "_stars": 187, "_contributors": [ { "user": "lmullen", "count": 175, "uuid": 183672 }, { "user": "dselivanov", "count": 6, "uuid": 5123805 }, { "user": "kbenoit", "count": 4, "uuid": 2182246 }, { "user": "jrnold", "count": 4, "uuid": 123968 }, { "user": "chrismuir", "count": 1, "uuid": 13386824 }, { "user": "emilhvitfeldt", "count": 1, "uuid": 14034784 }, { "user": "hideaki", "count": 1, "uuid": 19518 }, { "user": "jeroen", "count": 1, "uuid": 216319 }, { "user": "juliasilge", "count": 1, "uuid": 12505835 }, { "user": "karthik", "count": 1, "uuid": 138494 }, { "user": "maelle", "count": 1, "uuid": 8360597 }, { "user": "ironholds", "count": 1, "uuid": 2487262 }, { "user": "thomaschln", "count": 1, "uuid": 2394508 } ], "_userbio": { "uuid": 1200269, "type": "organization", "name": "rOpenSci", "followers": 1134, "description": "Tools and R Packages for Open Science" }, "_downloads": { "count": 30859, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/tokenizers" }, "_mentions": 1, "_devurl": "https://github.com/ropensci/tokenizers", "_pkgdown": "https://docs.ropensci.org/tokenizers/", "_searchresults": 1544, "_metadata": { "review": { "id": 33, "status": "reviewed", "version": "0.1.1", "organization": "rOpenSci Software Review", "url": "https://github.com/ropensci/software-review/issues/33" }, "ropensci_category": "scalereprod" }, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "extra/tokenizers.html", "LICENSE", "manual.pdf" ], "_homeurl": "https://github.com/ropensci/tokenizers", "_realowner": "ropensci", "_cranurl": true, "_releases": [ { "version": "0.1.0", "date": "2016-04-02" }, { "version": "0.1.1", "date": "2016-04-04" }, { "version": "0.1.2", "date": "2016-04-14" }, { "version": "0.1.3", "date": "2016-08-18" }, { "version": "0.1.4", "date": "2016-08-29" }, { "version": "0.2.0", "date": "2018-03-21" }, { "version": "0.2.1", "date": "2018-03-29" }, { "version": "0.2.3", "date": "2022-09-23" }, { "version": "0.3.0", "date": "2022-12-22" } ], "_exports": [ "chunk_text", "count_characters", "count_sentences", "count_words", "tokenize_character_shingles", "tokenize_characters", "tokenize_lines", "tokenize_ngrams", "tokenize_paragraphs", "tokenize_ptb", "tokenize_regex", "tokenize_sentences", "tokenize_skip_ngrams", "tokenize_word_stems", "tokenize_words" ], "_datasets": [ { "name": "mobydick", "title": "The text of Moby Dick", "object": "mobydick", "class": [ "character" ], "fields": [], "table": false, "tojson": true } ], "_help": [ { "page": "basic-tokenizers", "title": "Basic tokenizers", "topics": [ "basic-tokenizers", "tokenize_characters", "tokenize_lines", "tokenize_paragraphs", "tokenize_regex", "tokenize_sentences", "tokenize_words" ] }, { "page": "chunk_text", "title": "Chunk text into smaller segments", "topics": [ "chunk_text" ] }, { "page": "word-counting", "title": "Count words, sentences, characters", "topics": [ "count_characters", "count_sentences", "count_words" ] }, { "page": "mobydick", "title": "The text of Moby Dick", "topics": [ "mobydick" ] }, { "page": "ngram-tokenizers", "title": "N-gram tokenizers", "topics": [ "ngram-tokenizers", "tokenize_ngrams", "tokenize_skip_ngrams" ] }, { "page": "shingle-tokenizers", "title": "Character shingle tokenizers", "topics": [ "tokenize_character_shingles" ] }, { "page": "ptb-tokenizer", "title": "Penn Treebank Tokenizer", "topics": [ "tokenize_ptb" ] }, { "page": "stem-tokenizers", "title": "Word stem tokenizer", "topics": [ "tokenize_word_stems" ] }, { "page": "tokenizers", "title": "Tokenizers", "topics": [ "tokenizers-package", "tokenizers" ] } ], "_readme": "https://github.com/ropensci/tokenizers/raw/master/README.md", "_rundeps": [ "Rcpp", "SnowballC", "stringi" ], "_sysdeps": [ { "shlib": "libstdc++", "package": "libstdc++6", "source": "gcc", "version": "16-20260322-1ubuntu1", "name": "c++", "homepage": "http://gcc.gnu.org/", "description": "GNU Standard C++ Library v3" } ], "_vignettes": [ { "source": "introduction-to-tokenizers.Rmd", "filename": "introduction-to-tokenizers.html", "title": "Introduction to the tokenizers Package", "author": "Lincoln Mullen", "engine": "knitr::rmarkdown", "headings": [ "Package overview", "Character and character-shingle tokenizers", "Word and word-stem tokenizers", "N-gram and skip n-gram tokenizers", "Sentence and paragraph tokenizers", "Text chunking", "Counting words, characters, sentences" ], "created": "2016-08-11 20:12:37", "modified": "2022-12-19 21:14:10", "commits": 12 }, { "source": "tif-and-tokenizers.Rmd", "filename": "tif-and-tokenizers.html", "title": "The Text Interchange Formats and the tokenizers Package", "author": "Lincoln Mullen", "engine": "knitr::rmarkdown", "headings": [], "created": "2018-03-14 00:10:35", "modified": "2022-09-23 18:07:51", "commits": 5 } ], "_score": 13.403800343037906, "_indexed": true, "_nocasepkg": "tokenizers", "_universes": [ "ropensci", "thomaschln" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.3.1", "date": "2026-08-01T11:43:04.000Z", "distro": "resolute", "arch": "aarch64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/6aff61dc403ab2ee22dd3747650c75a6e6456e769847b1e77b08d726d8580ab9", "status": "success", "check": "NOTE", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.7.0", "os": "linux", "version": "0.3.1", "date": "2026-08-01T11:43:31.000Z", "distro": "resolute", "arch": "x86_64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/29a3fb977d2a8c5e06510fccfec1780cba1ba62a6e1e2e1f421b58ca7cc36930", "status": "success", "check": "NOTE", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.6.1", "os": "linux", "version": "0.3.1", "date": "2026-08-01T11:43:05.000Z", "distro": "resolute", "arch": "aarch64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/bd94fd2422d98b4b2896cd46bbb1547d0704f1186dcd75a24819cb6ddb877256", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.6.1", "os": "linux", "version": "0.3.1", "date": "2026-08-01T11:43:20.000Z", "distro": "resolute", "arch": "x86_64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/cdbff006c47a54bdd6a70f9d1fc24dd56fe39a97cd1f0c8df5d46d02ae812b0a", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.5.3", "os": "mac", "version": "0.3.1", "date": "2026-08-01T11:54:35.000Z", "arch": "aarch64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/c0a8e5feed1b6cc6d60e267bf11fc4781680700b86063dbf35d2f47fe7d586fc", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.5.3", "os": "mac", "version": "0.3.1", "date": "2026-08-01T11:56:30.000Z", "arch": "x86_64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/2738619623a8b901e9d62ef9e7c931a17f50c93c1ecab043ba6eb38443d4496d", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.6.1", "os": "mac", "version": "0.3.1", "date": "2026-08-01T11:55:13.000Z", "arch": "aarch64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/db3316817869686ee9b236de1d4d099a24df63572a027a5d32f19abe9f739f7e", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.6.1", "os": "mac", "version": "0.3.1", "date": "2026-08-01T11:56:51.000Z", "arch": "x86_64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/a2832da33337094faa500bd2460575f137a07101f3544f93eee3f6ac77b2a56b", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.6.0", "os": "wasm", "version": "0.3.1", "date": "2026-08-01T11:43:18.000Z", "arch": "emscripten", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/b275ae3d2e4258b6a5532b8cbe44c89341f35c083e9151064c68e7e7b3af2842", "status": "success", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.7.0", "os": "win", "version": "0.3.1", "date": "2026-08-01T11:43:57.000Z", "arch": "aarch64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/d9133e75168c1550bf5ec4641f5edf658491a8600ebf7b87c8acebba6752efaf", "status": "success", "check": "NOTE", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.7.0", "os": "win", "version": "0.3.1", "date": "2026-08-01T11:42:11.000Z", "arch": "x86_64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/e5f6da6ecb53f63b3fef511c3625dac75d640d08fa720723bcc9db7495a0fbb4", "status": "success", "check": "NOTE", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.5.3", "os": "win", "version": "0.3.1", "date": "2026-08-01T11:42:23.000Z", "arch": "x86_64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/ad3c1f9db7de4d2eb9178237f2844f5dc8e2d5c1360ebc546b99273113d481de", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.6.1", "os": "win", "version": "0.3.1", "date": "2026-08-01T11:44:34.000Z", "arch": "aarch64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/7b761e25f71959d6621110ff2a9778bf2430e2e0d9e922546223b330c40d290d", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" }, { "r": "4.6.1", "os": "win", "version": "0.3.1", "date": "2026-08-01T11:42:16.000Z", "arch": "x86_64", "commit": "b80863d088d4b39695b602ca11e061ac34770ec7", "fileid": "https://r2.ropensci.org/b63912df343f92539766c95f0f07162cb026347d582a0136045977823801f360", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/ropensci/actions/runs/30698039552" } ] }