195122c8d23154fe79d3525150f360d5 *DESCRIPTION
dc86c093012c1c9bc880868b901be232 *LICENSE
58bc95c04d9a4a010790e81bdc31d872 *LICENSE.note
58d9e22276cf763469ce74123fff1916 *NAMESPACE
dcd6c0d412b9e65b468a3bcbc54ca26e *NEWS.md
4a5b71cfd4fa08ca50304a7cd3da6cc3 *R/decoders.R
8a3e4798d33e4cfd4d87bf442a3ff5aa *R/encoding.R
de851b63434f5e778b712a7bb136628f *R/extendr-wrappers.R
707f1fc798331be1ecf069398a3c0f52 *R/models.R
4c655db57131a1721363916808df14de *R/normalizers.R
581d066eda915aef3f4fe60a70f8ea86 *R/post_processors.R
d27088cbe26d4ad7aa76697d3d4f8574 *R/pre_tokenizers.R
b6105f225aa70eb19fe34e4cdcffc51e *R/tokenizer.R
e06a7ea7000f6ad2ed0a5df60190bbf4 *R/trainers.R
5b09c83fbf9a930733e897c60652d6ee *README.md
ab6818dcb9d72a6e145d908496444c19 *inst/AUTHORS
fc5c865d20fb1374514b7a7b2c79db72 *man/decoder_byte_level.Rd
f53249ee287578ae7f2f4ccf5e6b6e58 *man/encoding.Rd
db2a33f6da1c375a45c2fb460835cf10 *man/model_bpe.Rd
95a483956014ebaf7ba3376ca43439f0 *man/model_unigram.Rd
58ee54b034899dc580467f2656d10cdf *man/model_wordpiece.Rd
8ff8b5ad03a70de364e65beef6225f6a *man/normalizer_nfc.Rd
5eb6a22f8452904c5eaf483656268be8 *man/normalizer_nfkc.Rd
826ede43be3374bf68d5766b2dd62565 *man/pre_tokenizer.Rd
c9734b1d815ea73fbeefa7af0cc6c6ff *man/pre_tokenizer_byte_level.Rd
5bf53a1606c37f3e9525128961387aef *man/pre_tokenizer_whitespace.Rd
78b245cee42feeaa504876debb17dbfc *man/processor_byte_level.Rd
d706fbb188ffe38df592307b37c82f6a *man/tok-package.Rd
2e0ed460d6efe7d06e9276c1c21bce0a *man/tok_decoder.Rd
d0198a97f723ce4dfbe6ef4dbc53571e *man/tok_model.Rd
b2f8e5780581e6e05b0f7f5c546036b8 *man/tok_normalizer.Rd
6c968375a9926b5f42e499698db9aed4 *man/tok_processor.Rd
d1ccc7c4ae5804d9456eb2b656067e6c *man/tok_trainer.Rd
81e7f1e26f9d774a604aa4d190ac8358 *man/tokenizer.Rd
420bcd57270067bb8ca1b6eb11d3cbc8 *man/trainer_bpe.Rd
0c75314c63f5ee249aabf4f741bc573c *man/trainer_unigram.Rd
e0107b09c4dcd8ddb4596f7da4e6882b *man/trainer_wordpiece.Rd
1d1e02ff0bcb100dca97d231b70ae8da *src/Makevars
27957123fba16c05c53a66e68b0f3482 *src/Makevars.ucrt
4d12fbf066fb357db1e202bfc7f60bab *src/Makevars.win
e475c45acff0aec8451fa9e0d2ade36c *src/entrypoint.c
91ffb80d785800539d25081994e9a358 *src/rust/Cargo.lock
b189c5c0b29fbd4f868d42d66341117e *src/rust/Cargo.toml
f47439b192b8ccb1f1333e93291ef128 *src/rust/config/cargo-config.toml
f79a98417550c5f434aa3ca8d0a809cb *src/rust/src/decoders.rs
0458be593b6569ecdd65dc527a5c4a2e *src/rust/src/lib.rs
e407d3bc7b63a5de6330183ce8b21deb *src/rust/src/models.rs
e3bbc5d34a774898b7013c0d53def544 *src/rust/src/normalizers.rs
7b31d92e231058940a75bc0fbca5edb1 *src/rust/src/post_processors.rs
50fb19aae42b9f9925f45cc2aa298e9c *src/rust/src/pre_tokenizers.rs
20c7eeea3e628f0d30de4dfea4787e8f *src/rust/src/tokenizer.rs
558fde248c4e122f19c2ed2f9cee995f *src/rust/src/trainers.rs
7b7fd4014f405edcda71f04b77c1f2c0 *src/rust/vendor.tar.xz
ba398b93085f305afd0f06437e58899f *src/tok-win.def
2fa9c1f6e421b6a79ceeee8fc410730b *tests/testthat.R
a379162c68dc3aac382a2d98facf4c44 *tests/testthat/assets/tokenizer.json
94ea82db26368d89a4fe5bc69f6e7de6 *tests/testthat/test-decoders.R
dcf23f7ae3377b349c5c54906d460a8b *tests/testthat/test-encoding.R
e298050c9b2e5a47c1299ac438cde5a0 *tests/testthat/test-models.R
9bf4630914215ace43292d7e8f617c0a *tests/testthat/test-normalizers.R
d94a3ac8e8f1d828859ae0715b7b2fa0 *tests/testthat/test-post_processors.R
9cd4d8e57905d61a992b0d9945d48f90 *tests/testthat/test-pre_tokenizers.R
2847186a0d8b2fab5e187e5174fd6338 *tests/testthat/test-tokenizer.R
839814871c136e1452f80d76d68af077 *tests/testthat/test-trainers.R
9da9f44f6b828277200c9971eebe15de *tools/patch.R
cd2990ec9b81ab0b80f56a753831fbac *tools/update_authors.R
caa74dadd0737705d92a72e8f37cb51e *tools/vendor.sh
