From 519cc13be0db9394eac36ae93e74df5456bcface Mon Sep 17 00:00:00 2001 From: h-vetinari Date: Fri, 6 May 2022 00:48:40 +1100 Subject: [PATCH] Upgrade pyo3 to 0.16 (#956) * Upgrade pyo3 to 0.15 Rebase-conflicts-fixed-by: H. Vetinari * Upgrade pyo3 to 0.16 Rebase-conflicts-fixed-by: H. Vetinari * Install Python before running cargo clippy * Fix clippy warnings * Use `PyArray_Check` instead of downcasting to `PyArray1` * Enable `auto-initialize` of pyo3 to fix `cargo test --no-default-features` * Fix some test cases Why do they change? * Refactor and add SAFETY comments to `PyArrayUnicode` Replace deprecated `PyUnicode_FromUnicode` with `PyUnicode_FromKindAndData` Co-authored-by: messense --- .github/workflows/python.yml | 13 +- bindings/python/.cargo/config.toml | 1 - bindings/python/Cargo.lock | 681 +++++++++---------- bindings/python/Cargo.toml | 7 +- bindings/python/src/decoders.rs | 28 +- bindings/python/src/encoding.rs | 57 +- bindings/python/src/error.rs | 2 +- bindings/python/src/lib.rs | 2 +- bindings/python/src/models.rs | 44 +- bindings/python/src/normalizers.rs | 62 +- bindings/python/src/pre_tokenizers.rs | 59 +- bindings/python/src/processors.rs | 31 +- bindings/python/src/token.rs | 2 +- bindings/python/src/tokenizer.rs | 144 ++-- bindings/python/src/trainers.rs | 22 +- bindings/python/src/utils/iterators.rs | 2 +- bindings/python/src/utils/normalization.rs | 51 +- bindings/python/src/utils/pretokenization.rs | 16 +- bindings/python/src/utils/regex.rs | 4 +- 19 files changed, 614 insertions(+), 614 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index d3814115f..08fb23484 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -63,6 +63,13 @@ jobs: toolchain: stable components: rustfmt, clippy + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + architecture: "x64" + + - name: Cache Cargo Registry uses: actions/cache@v1 with: @@ -88,12 +95,6 @@ jobs: command: clippy args: --manifest-path ./bindings/python/Cargo.toml --all-targets --all-features -- -D warnings - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - architecture: "x64" - - name: Install working-directory: ./bindings/python run: | diff --git a/bindings/python/.cargo/config.toml b/bindings/python/.cargo/config.toml index e91cd8b97..dd4042827 100644 --- a/bindings/python/.cargo/config.toml +++ b/bindings/python/.cargo/config.toml @@ -11,4 +11,3 @@ rustflags = [ "-C", "link-arg=dynamic_lookup", "-C", "link-arg=-mmacosx-version-min=10.11", ] - diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 2fdcfc1ef..ff5b74a17 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -19,9 +19,9 @@ dependencies = [ [[package]] name = "ansi_term" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ "winapi", ] @@ -45,9 +45,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" @@ -69,9 +69,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitvec" -version = "0.19.5" +version = "0.19.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321" +checksum = "55f93d0ef3363c364d5976646a38f04cf67cfe1d4c8d160cdea02cab2c116b33" dependencies = [ "funty", "radium", @@ -81,18 +81,18 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ "generic-array", ] [[package]] name = "bumpalo" -version = "3.7.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" [[package]] name = "byteorder" @@ -129,16 +129,16 @@ dependencies = [ [[package]] name = "cached-path" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2e84e6a7366ce5a2d7ce7fb466610d4d906d89981bc93215e215937a2c51ce" +checksum = "5f1c56d30236522ab3393a08746b138d4e16372001f42d29c88d513aeb8ab7ef" dependencies = [ "flate2", "fs2", "glob", - "indicatif", + "indicatif 0.16.2", "log", - "rand 0.8.4", + "rand 0.8.5", "reqwest", "serde", "serde_json", @@ -152,15 +152,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0" - -[[package]] -name = "cfg-if" -version = "0.1.10" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" [[package]] name = "cfg-if" @@ -170,9 +164,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "2.33.3" +version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", @@ -185,13 +179,13 @@ dependencies = [ [[package]] name = "console" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45" +checksum = "a28b32d32ca44b70c3e4acd7db1babf555fa026e385fb95f18028f88848b3c31" dependencies = [ "encode_unicode", - "lazy_static", "libc", + "once_cell", "regex", "terminal_size", "unicode-width", @@ -200,9 +194,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -210,35 +204,35 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.1" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -248,18 +242,19 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.5" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ - "cfg-if 1.0.0", + "autocfg", + "cfg-if", "crossbeam-utils", "lazy_static", "memoffset", @@ -268,22 +263,22 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "lazy_static", ] [[package]] -name = "ctor" -version = "0.1.21" +name = "crypto-common" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccc0a48a9b826acdf4028595adc9db92caea352f7af011a3034acd172a52a0aa" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" dependencies = [ - "quote", - "syn", + "generic-array", + "typenum", ] [[package]] @@ -348,11 +343,12 @@ dependencies = [ [[package]] name = "digest" -version = "0.9.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "generic-array", + "block-buffer", + "crypto-common", ] [[package]] @@ -366,9 +362,9 @@ dependencies = [ [[package]] name = "dirs-sys" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", @@ -389,11 +385,11 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.28" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -418,13 +414,22 @@ dependencies = [ "cc", ] +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + [[package]] name = "filetime" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall", "winapi", @@ -432,11 +437,11 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80edafed416a46fb378521624fab1cfa2eb514784fd8921adbe8a8d8321da811" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crc32fast", "libc", "miniz_oxide", @@ -491,44 +496,43 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "futures-channel" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-sink" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" [[package]] name = "futures-task" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" [[package]] name = "futures-util" -version = "0.3.17" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ - "autocfg", "futures-core", "futures-io", "futures-task", @@ -540,9 +544,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.4" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" dependencies = [ "typenum", "version_check", @@ -554,33 +558,22 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] name = "getrandom" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.10.2+wasi-snapshot-preview1", ] -[[package]] -name = "ghost" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5bcf1bbeab73aa4cf2fde60a846858dc036163c7c33bec309f8d17de785479" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "glob" version = "0.3.0" @@ -589,9 +582,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "h2" -version = "0.3.4" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472" +checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" dependencies = [ "bytes", "fnv", @@ -623,9 +616,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11" +checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" dependencies = [ "bytes", "fnv", @@ -634,9 +627,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" dependencies = [ "bytes", "http", @@ -645,15 +638,15 @@ dependencies = [ [[package]] name = "httparse" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" +checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" [[package]] name = "httpdate" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "humantime" @@ -666,9 +659,9 @@ dependencies = [ [[package]] name = "hyper" -version = "0.14.12" +version = "0.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f67199e765030fa08fe0bd581af683f0d5bc04ea09c2b1102012c5fb90e7fd" +checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd" dependencies = [ "bytes", "futures-channel", @@ -720,9 +713,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" dependencies = [ "autocfg", "hashbrown", @@ -736,69 +729,45 @@ checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" dependencies = [ "console", "lazy_static", - "number_prefix", + "number_prefix 0.3.0", "regex", ] [[package]] -name = "indoc" -version = "0.3.6" +name = "indicatif" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47741a8bc60fb26eb8d6e0238bbb26d8575ff623fdc97b1a2c00c050b9684ed8" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" dependencies = [ - "indoc-impl", - "proc-macro-hack", + "console", + "lazy_static", + "number_prefix 0.4.0", + "regex", ] [[package]] -name = "indoc-impl" -version = "0.3.6" +name = "indoc" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce046d161f000fffde5f432a0d034d0341dc152643b2598ed5bfce44c4f3a8f0" +checksum = "e7906a9fababaeacb774f72410e497a1d18de916322e33797bb2cd29baa23c9e" dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "syn", "unindent", ] [[package]] name = "instant" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "inventory" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0f7efb804ec95e33db9ad49e4252f049e37e8b0a4652e3cd61f7999f2eff7f" -dependencies = [ - "ctor", - "ghost", - "inventory-impl", -] - -[[package]] -name = "inventory-impl" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75c094e94816723ab936484666968f5b58060492e880f3c8d00489a1e244fa51" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "proc-macro2", - "quote", - "syn", + "cfg-if", ] [[package]] name = "ipnet" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" [[package]] name = "itertools" @@ -820,15 +789,15 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.8" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" [[package]] name = "js-sys" -version = "0.3.53" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4bf49d50e2961077d9c99f4b7997d770a1114f087c3c2e0069b36c13fc2979d" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" dependencies = [ "wasm-bindgen", ] @@ -847,33 +816,33 @@ checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec", "bitflags", - "cfg-if 1.0.0", + "cfg-if", "ryu", "static_assertions", ] [[package]] name = "libc" -version = "0.2.101" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" +checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" [[package]] name = "lock_api" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" dependencies = [ "scopeguard", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "1c4dcd960cc540667f619483fc99102f88d6118b87730e24e8fbe8054b7445e4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -906,6 +875,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "matrixmultiply" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add85d4dd35074e6fedc608f8c8f513a3548619a9024b751949ef0e8e45a4d84" +dependencies = [ + "rawpointer", +] + [[package]] name = "memchr" version = "2.3.4" @@ -914,9 +892,9 @@ checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" [[package]] name = "memoffset" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] @@ -939,14 +917,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.13" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" dependencies = [ "libc", "log", "miow", "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi", ] @@ -983,8 +962,21 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac06db03ec2f46ee0ecdca1a1c34a99c0d188a0d83439b84bf0cb4b386e4ab09" dependencies = [ - "matrixmultiply", - "num-complex", + "matrixmultiply 0.2.4", + "num-complex 0.2.4", + "num-integer", + "num-traits", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec23e6762830658d2b3d385a75aa212af2f67a4586d4442907144f3bb6a1ca8" +dependencies = [ + "matrixmultiply 0.3.2", + "num-complex 0.4.0", "num-integer", "num-traits", "rawpointer", @@ -1005,9 +997,9 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" dependencies = [ "winapi", ] @@ -1022,6 +1014,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085" +dependencies = [ + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -1043,9 +1044,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", @@ -1057,31 +1058,36 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "numpy" -version = "0.12.2" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd9e8e652becf4ba6c11803945f8bf463c23f482f704bb33f70ae9d22482d10" +checksum = "383ae168529a39fc97cbc1d9d4fa865377731a519bc27553ed96f50594de7c45" dependencies = [ - "cfg-if 0.1.10", "libc", - "ndarray", - "num-complex", + "ndarray 0.15.4", + "num-complex 0.4.0", "num-traits", "pyo3", ] [[package]] name = "once_cell" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "onig" -version = "6.2.0" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16fd3c0e73b516af509c13c4ba76ec0c987ce20d78b38cff356b8d01fc6a6c0" +checksum = "67ddfe2c93bb389eea6e6d713306880c7f6dcc99a75b659ce145d962c861b225" dependencies = [ "bitflags", "lazy_static", @@ -1091,28 +1097,22 @@ dependencies = [ [[package]] name = "onig_sys" -version = "69.7.0" +version = "69.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fd9442a09e4fbd08d196ddf419b2c79a43c3a46c800320cc841d45c2449a240" +checksum = "5dd3eee045c84695b53b20255bb7317063df090b68e18bfac0abb6c39cf7f33e" dependencies = [ "cc", "pkg-config", ] -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" -version = "0.10.36" +version = "0.10.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a" +checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" dependencies = [ "bitflags", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -1121,15 +1121,15 @@ dependencies = [ [[package]] name = "openssl-probe" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.66" +version = "0.9.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1996d2d305e561b70d1ee0c53f1542833f4e1ac6ce9a6708b6ff2738ca67dc82" +checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" dependencies = [ "autocfg", "cc", @@ -1155,7 +1155,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "instant", "libc", "redox_syscall", @@ -1163,31 +1163,12 @@ dependencies = [ "winapi", ] -[[package]] -name = "paste" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" -dependencies = [ - "paste-impl", - "proc-macro-hack", -] - [[package]] name = "paste" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" -[[package]] -name = "paste-impl" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" -dependencies = [ - "proc-macro-hack", -] - [[package]] name = "percent-encoding" version = "2.1.0" @@ -1196,9 +1177,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "pin-project-lite" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" +checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" [[package]] name = "pin-utils" @@ -1208,65 +1189,80 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.19" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" [[package]] name = "ppv-lite86" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" - -[[package]] -name = "proc-macro-hack" -version = "0.5.19" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "proc-macro2" -version = "1.0.29" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" dependencies = [ "unicode-xid", ] [[package]] name = "pyo3" -version = "0.12.4" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6bbbe8f70d179260b3728e5d04eb012f4f0c7988e58c11433dd689cecaa72e" +checksum = "a378727d5fdcaafd15b5afe9842cff1c25fdc43f62a162ffda2263c57ad98703" dependencies = [ - "ctor", + "cfg-if", "indoc", - "inventory", "libc", "parking_lot", - "paste 0.1.18", - "pyo3cls", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", "unindent", ] [[package]] -name = "pyo3-derive-backend" -version = "0.12.4" +name = "pyo3-build-config" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbb27a3e96edd34c13d97d0feefccc90a79270c577c66e19d95af8323823dfc" +dependencies = [ + "once_cell", +] + +[[package]] +name = "pyo3-ffi" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10ecd0eb6ed7b3d9965b4f4370b5b9e99e3e5e8742000e1c452c018f8c2a322f" +checksum = "7b719fff844bcf3f911132112ec06527eb195f6a98e0c42cf97e1118929fd4ea" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f795e52d3320abb349ca28b501a7112154a87f353fae1c811deecd58e99cfa9b" dependencies = [ "proc-macro2", + "pyo3-macros-backend", "quote", "syn", ] [[package]] -name = "pyo3cls" -version = "0.12.4" +name = "pyo3-macros-backend" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d344fdaa6a834a06dd1720ff104ea12fe101dad2e8db89345af9db74c0bb11a0" +checksum = "39e03aa57a3bb7b96982958088df38302a139df4eef54671bc595f26556cb75b" dependencies = [ - "pyo3-derive-backend", + "proc-macro2", + "pyo3-build-config", "quote", "syn", ] @@ -1279,9 +1275,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.9" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" dependencies = [ "proc-macro2", ] @@ -1302,19 +1298,18 @@ dependencies = [ "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", - "rand_hc 0.2.0", + "rand_hc", ] [[package]] name = "rand" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", "rand_core 0.6.3", - "rand_hc 0.3.1", ] [[package]] @@ -1352,7 +1347,7 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.3", + "getrandom 0.2.5", ] [[package]] @@ -1364,15 +1359,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core 0.6.3", -] - [[package]] name = "rawpointer" version = "0.2.1" @@ -1417,21 +1403,22 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" dependencies = [ - "getrandom 0.2.3", + "getrandom 0.2.5", "redox_syscall", + "thiserror", ] [[package]] @@ -1462,15 +1449,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.4" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" dependencies = [ "base64 0.13.0", "bytes", "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "hyper", @@ -1484,6 +1472,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "serde", + "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", @@ -1496,9 +1485,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" [[package]] name = "schannel" @@ -1518,9 +1507,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "security-framework" -version = "2.4.2" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ "bitflags", "core-foundation", @@ -1531,9 +1520,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.4.2" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" dependencies = [ "core-foundation-sys", "libc", @@ -1541,18 +1530,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.130" +version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.130" +version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" dependencies = [ "proc-macro2", "quote", @@ -1561,9 +1550,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.67" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" dependencies = [ "itoa", "ryu", @@ -1572,9 +1561,9 @@ dependencies = [ [[package]] name = "serde_urlencoded" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", "itoa", @@ -1584,34 +1573,32 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.6" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9204c41a1597a8c5af23c82d1c921cb01ec0a4c59e07a9c7306062829a3903f3" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ - "block-buffer", - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", - "opaque-debug", ] [[package]] name = "slab" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" [[package]] name = "smallvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" [[package]] name = "socket2" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" dependencies = [ "libc", "winapi", @@ -1649,9 +1636,9 @@ checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" [[package]] name = "syn" -version = "1.0.76" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84" +checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" dependencies = [ "proc-macro2", "quote", @@ -1666,9 +1653,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f5515d3add52e0bbdcad7b83c388bb36ba7b754dda3b5f5bc2d38640cdba5c" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" dependencies = [ "filetime", "libc", @@ -1677,13 +1664,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", + "fastrand", "libc", - "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi", @@ -1691,9 +1678,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] @@ -1749,9 +1736,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.3.1" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" dependencies = [ "tinyvec_macros", ] @@ -1772,13 +1759,13 @@ dependencies = [ "derive_builder", "dirs", "esaxx-rs", - "indicatif", + "indicatif 0.15.0", "itertools 0.9.0", "lazy_static", "log", "macro_rules_attribute", "onig", - "paste 1.0.6", + "paste", "rand 0.7.3", "rayon", "rayon-cond", @@ -1801,7 +1788,7 @@ dependencies = [ "env_logger", "itertools 0.9.0", "libc", - "ndarray", + "ndarray 0.13.1", "numpy", "onig", "pyo3", @@ -1814,17 +1801,17 @@ dependencies = [ [[package]] name = "tokio" -version = "1.11.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4efe6fc2395938c8155973d7be49fe8d03a843726e285e100a8a383cc0154ce" +checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" dependencies = [ - "autocfg", "bytes", "libc", "memchr", "mio", "num_cpus", "pin-project-lite", + "socket2", "winapi", ] @@ -1840,9 +1827,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d3725d3efa29485e87311c5b699de63cde14b00ed4d256b8318aa30ca452cd" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ "bytes", "futures-core", @@ -1860,20 +1847,20 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.26" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d" +checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "pin-project-lite", "tracing-core", ] [[package]] name = "tracing-core" -version = "0.1.19" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8" +checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" dependencies = [ "lazy_static", ] @@ -1886,15 +1873,15 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "unicode-bidi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" [[package]] name = "unicode-normalization" @@ -1916,15 +1903,15 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" [[package]] name = "unicode-width" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" [[package]] name = "unicode-xid" @@ -1940,9 +1927,9 @@ checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "unindent" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +checksum = "514672a55d7380da379785a4d70ca8386c8883ff7eaae877be4d2081cebe73d8" [[package]] name = "url" @@ -1970,9 +1957,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" @@ -1996,23 +1983,27 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" -version = "0.2.76" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce9b1b516211d33767048e5d47fa2a381ed8b76fc48d2ce4aa39877f9f183e0" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" dependencies = [ - "cfg-if 1.0.0", - "serde", - "serde_json", + "cfg-if", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.76" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe8dc78e2326ba5f845f4b5bf548401604fa20b1dd1d365fb73b6c1d6364041" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" dependencies = [ "bumpalo", "lazy_static", @@ -2025,11 +2016,11 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.26" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fded345a6559c2cfee778d562300c581f7d4ff3edb9b0d230d69800d213972" +checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -2037,9 +2028,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.76" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44468aa53335841d9d6b6c023eaab07c0cd4bddbcfdee3e2bb1e8d2cb8069fef" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2047,9 +2038,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.76" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0195807922713af1e67dc66132c7328206ed9766af3858164fb583eedc25fbad" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" dependencies = [ "proc-macro2", "quote", @@ -2060,15 +2051,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.76" +version = "0.2.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdb075a845574a1fa5f09fd77e43f7747599301ea3417a9fbffdeedfc1f4a29" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" [[package]] name = "web-sys" -version = "0.3.53" +version = "0.3.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224b2f6b67919060055ef1a67807367c2066ed520c3862cc013d26cf893a783c" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" dependencies = [ "js-sys", "wasm-bindgen", @@ -2107,9 +2098,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winreg" -version = "0.7.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 10f434ee9..cebc70e16 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -14,8 +14,8 @@ serde = { version = "1.0", features = [ "rc", "derive" ]} serde_json = "1.0" libc = "0.2" env_logger = "0.7.1" -pyo3 = "0.12" -numpy = "0.12" +pyo3 = "0.16.2" +numpy = "0.16.2" ndarray = "0.13" onig = { version = "6.0", default-features = false } itertools = "0.9" @@ -26,8 +26,7 @@ path = "../../tokenizers" [dev-dependencies] tempfile = "3.1" +pyo3 = { version = "0.16.2", features = ["auto-initialize"] } [features] default = ["pyo3/extension-module"] - - diff --git a/bindings/python/src/decoders.rs b/bindings/python/src/decoders.rs index 5f15838cb..b793f3a0f 100644 --- a/bindings/python/src/decoders.rs +++ b/bindings/python/src/decoders.rs @@ -21,7 +21,7 @@ use super::error::ToPyResult; /// /// This class is not supposed to be instantiated directly. Instead, any implementation of /// a Decoder will return an instance of this class when instantiated. -#[pyclass(dict, module = "tokenizers.decoders", name=Decoder)] +#[pyclass(dict, module = "tokenizers.decoders", name = "Decoder", subclass)] #[derive(Clone, Deserialize, Serialize)] pub struct PyDecoder { #[serde(flatten)] @@ -97,7 +97,7 @@ impl PyDecoder { /// /// Returns: /// :obj:`str`: The decoded string - #[text_signature = "(self, tokens)"] + #[pyo3(text_signature = "(self, tokens)")] fn decode(&self, tokens: Vec) -> PyResult { ToPyResult(self.decoder.decode(tokens)).into() } @@ -141,8 +141,8 @@ macro_rules! setter { /// /// This decoder is to be used in tandem with the :class:`~tokenizers.pre_tokenizers.ByteLevel` /// :class:`~tokenizers.pre_tokenizers.PreTokenizer`. -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name=ByteLevel)] -#[text_signature = "(self)"] +#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "ByteLevel")] +#[pyo3(text_signature = "(self)")] pub struct PyByteLevelDec {} #[pymethods] impl PyByteLevelDec { @@ -161,8 +161,8 @@ impl PyByteLevelDec { /// cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): /// Whether to cleanup some tokenization artifacts. Mainly spaces before punctuation, /// and some abbreviated english forms. -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name=WordPiece)] -#[text_signature = "(self, prefix=\"##\", cleanup=True)"] +#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "WordPiece")] +#[pyo3(text_signature = "(self, prefix=\"##\", cleanup=True)")] pub struct PyWordPieceDec {} #[pymethods] impl PyWordPieceDec { @@ -203,8 +203,8 @@ impl PyWordPieceDec { /// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): /// Whether to add a space to the first word if there isn't already one. This /// lets us treat `hello` exactly like `say hello`. -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name=Metaspace)] -#[text_signature = "(self, replacement = \"▁\", add_prefix_space = True)"] +#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "Metaspace")] +#[pyo3(text_signature = "(self, replacement = \"▁\", add_prefix_space = True)")] pub struct PyMetaspaceDec {} #[pymethods] impl PyMetaspaceDec { @@ -244,8 +244,8 @@ impl PyMetaspaceDec { /// suffix (:obj:`str`, `optional`, defaults to :obj:``): /// The suffix that was used to caracterize an end-of-word. This suffix will /// be replaced by whitespaces during the decoding -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name=BPEDecoder)] -#[text_signature = "(self, suffix=\"\")"] +#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "BPEDecoder")] +#[pyo3(text_signature = "(self, suffix=\"\")")] pub struct PyBPEDecoder {} #[pymethods] impl PyBPEDecoder { @@ -276,8 +276,8 @@ impl PyBPEDecoder { /// cleanup (:obj:`bool`, `optional`, defaults to :obj:`True`): /// Whether to cleanup some tokenization artifacts. /// Mainly spaces before punctuation, and some abbreviated english forms. -#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name=CTC)] -#[text_signature = "(self, pad_token=\"\", word_delimiter_token=\"|\", cleanup=True)"] +#[pyclass(extends=PyDecoder, module = "tokenizers.decoders", name = "CTC")] +#[pyo3(text_signature = "(self, pad_token=\"\", word_delimiter_token=\"|\", cleanup=True)")] pub struct PyCTCDecoder {} #[pymethods] impl PyCTCDecoder { @@ -420,8 +420,8 @@ mod test { let py_meta = py_dec.get_as_subtype().unwrap(); let gil = Python::acquire_gil(); assert_eq!( - "tokenizers.decoders.Metaspace", - py_meta.as_ref(gil.python()).get_type().name() + "Metaspace", + py_meta.as_ref(gil.python()).get_type().name().unwrap() ); } diff --git a/bindings/python/src/encoding.rs b/bindings/python/src/encoding.rs index 976b2c605..cadaccd43 100644 --- a/bindings/python/src/encoding.rs +++ b/bindings/python/src/encoding.rs @@ -1,7 +1,6 @@ use pyo3::exceptions; use pyo3::prelude::*; use pyo3::types::*; -use pyo3::{PyObjectProtocol, PySequenceProtocol}; use tk::tokenizer::{Offsets, PaddingDirection}; use tk::utils::truncation::TruncationDirection; use tokenizers as tk; @@ -9,7 +8,7 @@ use tokenizers as tk; use crate::error::{deprecation_warning, PyError}; /// The :class:`~tokenizers.Encoding` represents the output of a :class:`~tokenizers.Tokenizer`. -#[pyclass(dict, module = "tokenizers", name=Encoding)] +#[pyclass(dict, module = "tokenizers", name = "Encoding")] #[repr(transparent)] pub struct PyEncoding { pub encoding: tk::tokenizer::Encoding, @@ -21,24 +20,6 @@ impl From for PyEncoding { } } -#[pyproto] -impl PyObjectProtocol for PyEncoding { - fn __repr__(&self) -> PyResult { - Ok(format!( - "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \ - attention_mask, special_tokens_mask, overflowing])", - self.encoding.get_ids().len() - )) - } -} - -#[pyproto] -impl PySequenceProtocol for PyEncoding { - fn __len__(&self) -> PyResult { - Ok(self.encoding.len()) - } -} - #[pymethods] impl PyEncoding { #[new] @@ -73,6 +54,18 @@ impl PyEncoding { } } + fn __repr__(&self) -> PyResult { + Ok(format!( + "Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \ + attention_mask, special_tokens_mask, overflowing])", + self.encoding.get_ids().len() + )) + } + + fn __len__(&self) -> PyResult { + Ok(self.encoding.len()) + } + /// Merge the list of encodings into one final :class:`~tokenizers.Encoding` /// /// Args: @@ -86,7 +79,7 @@ impl PyEncoding { /// :class:`~tokenizers.Encoding`: The resulting Encoding #[staticmethod] #[args(growing_offsets = true)] - #[text_signature = "(encodings, growing_offsets=True)"] + #[pyo3(text_signature = "(encodings, growing_offsets=True)")] fn merge(encodings: Vec>, growing_offsets: bool) -> PyEncoding { tk::tokenizer::Encoding::merge( encodings.into_iter().map(|e| e.encoding.clone()), @@ -108,7 +101,7 @@ impl PyEncoding { /// /// Set the given sequence index for the whole range of tokens contained in this /// :class:`~tokenizers.Encoding`. - #[text_signature = "(self, sequence_id)"] + #[pyo3(text_signature = "(self, sequence_id)")] fn set_sequence_id(&mut self, sequence_id: usize) { self.encoding.set_sequence_id(sequence_id); } @@ -270,7 +263,7 @@ impl PyEncoding { /// Returns: /// :obj:`Tuple[int, int]`: The range of tokens: :obj:`(first, last + 1)` #[args(sequence_index = 0)] - #[text_signature = "(self, word_index, sequence_index=0)"] + #[pyo3(text_signature = "(self, word_index, sequence_index=0)")] fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> { self.encoding.word_to_tokens(word_index, sequence_index) } @@ -286,7 +279,7 @@ impl PyEncoding { /// Returns: /// :obj:`Tuple[int, int]`: The range of characters (span) :obj:`(first, last + 1)` #[args(sequence_index = 0)] - #[text_signature = "(self, word_index, sequence_index=0)"] + #[pyo3(text_signature = "(self, word_index, sequence_index=0)")] fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option { self.encoding.word_to_chars(word_index, sequence_index) } @@ -302,7 +295,7 @@ impl PyEncoding { /// /// Returns: /// :obj:`int`: The sequence id of the given token - #[text_signature = "(self, token_index)"] + #[pyo3(text_signature = "(self, token_index)")] fn token_to_sequence(&self, token_index: usize) -> Option { self.encoding.token_to_sequence(token_index) } @@ -319,7 +312,7 @@ impl PyEncoding { /// /// Returns: /// :obj:`Tuple[int, int]`: The token offsets :obj:`(first, last + 1)` - #[text_signature = "(self, token_index)"] + #[pyo3(text_signature = "(self, token_index)")] fn token_to_chars(&self, token_index: usize) -> Option { let (_, offsets) = self.encoding.token_to_chars(token_index)?; Some(offsets) @@ -337,7 +330,7 @@ impl PyEncoding { /// /// Returns: /// :obj:`int`: The index of the word in the relevant input sequence. - #[text_signature = "(self, token_index)"] + #[pyo3(text_signature = "(self, token_index)")] fn token_to_word(&self, token_index: usize) -> Option { let (_, word_idx) = self.encoding.token_to_word(token_index)?; Some(word_idx) @@ -354,7 +347,7 @@ impl PyEncoding { /// Returns: /// :obj:`int`: The index of the token that contains this char in the encoded sequence #[args(sequence_index = 0)] - #[text_signature = "(self, char_pos, sequence_index=0)"] + #[pyo3(text_signature = "(self, char_pos, sequence_index=0)")] fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option { self.encoding.char_to_token(char_pos, sequence_index) } @@ -370,7 +363,7 @@ impl PyEncoding { /// Returns: /// :obj:`int`: The index of the word that contains this char in the input sequence #[args(sequence_index = 0)] - #[text_signature = "(self, char_pos, sequence_index=0)"] + #[pyo3(text_signature = "(self, char_pos, sequence_index=0)")] fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option { self.encoding.char_to_word(char_pos, sequence_index) } @@ -393,7 +386,9 @@ impl PyEncoding { /// pad_token (:obj:`str`, defaults to `[PAD]`): /// The pad token to use #[args(kwargs = "**")] - #[text_signature = "(self, length, direction='right', pad_id=0, pad_type_id=0, pad_token='[PAD]')"] + #[pyo3( + text_signature = "(self, length, direction='right', pad_id=0, pad_type_id=0, pad_token='[PAD]')" + )] fn pad(&mut self, length: usize, kwargs: Option<&PyDict>) -> PyResult<()> { let mut pad_id = 0; let mut pad_type_id = 0; @@ -445,7 +440,7 @@ impl PyEncoding { /// Truncate direction #[args(stride = "0")] #[args(direction = "\"right\"")] - #[text_signature = "(self, max_length, stride=0, direction='right')"] + #[pyo3(text_signature = "(self, max_length, stride=0, direction='right')")] fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> { let tdir = match direction { "left" => Ok(TruncationDirection::Left), diff --git a/bindings/python/src/error.rs b/bindings/python/src/error.rs index a6bcaf3fe..1e8c5a1db 100644 --- a/bindings/python/src/error.rs +++ b/bindings/python/src/error.rs @@ -37,7 +37,7 @@ impl ToPyResult { pub(crate) fn deprecation_warning(version: &str, message: &str) -> PyResult<()> { let gil = pyo3::Python::acquire_gil(); let python = gil.python(); - let deprecation_warning = python.import("builtins")?.get("DeprecationWarning")?; + let deprecation_warning = python.import("builtins")?.getattr("DeprecationWarning")?; let full_message = format!("Deprecated in {}: {}", version, message); pyo3::PyErr::warn(python, deprecation_warning, &full_message, 0) } diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 947dbda2a..b1580b0e4 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -126,7 +126,7 @@ fn normalizers(_py: Python, m: &PyModule) -> PyResult<()> { /// Tokenizers Module #[pymodule] fn tokenizers(_py: Python, m: &PyModule) -> PyResult<()> { - env_logger::init_from_env("TOKENIZERS_LOG"); + let _ = env_logger::try_init_from_env("TOKENIZERS_LOG"); // Register the fork callback #[cfg(target_family = "unix")] diff --git a/bindings/python/src/models.rs b/bindings/python/src/models.rs index e7aaa79bc..e529332de 100644 --- a/bindings/python/src/models.rs +++ b/bindings/python/src/models.rs @@ -24,7 +24,7 @@ use super::error::{deprecation_warning, ToPyResult}; /// will contain and manage the learned vocabulary. /// /// This class cannot be constructed directly. Please use one of the concrete models. -#[pyclass(module = "tokenizers.models", name=Model)] +#[pyclass(module = "tokenizers.models", name = "Model", subclass)] #[derive(Clone, Serialize, Deserialize)] pub struct PyModel { #[serde(flatten)] @@ -132,7 +132,7 @@ impl PyModel { /// /// Returns: /// A :obj:`List` of :class:`~tokenizers.Token`: The generated tokens - #[text_signature = "(self, sequence)"] + #[pyo3(text_signature = "(self, sequence)")] fn tokenize(&self, sequence: &str) -> PyResult> { Ok(ToPyResult(self.model.read().unwrap().tokenize(sequence)) .into_py()? @@ -149,7 +149,7 @@ impl PyModel { /// /// Returns: /// :obj:`int`: The ID associated to the token - #[text_signature = "(self, tokens)"] + #[pyo3(text_signature = "(self, tokens)")] fn token_to_id(&self, token: &str) -> Option { self.model.read().unwrap().token_to_id(token) } @@ -162,7 +162,7 @@ impl PyModel { /// /// Returns: /// :obj:`str`: The token associated to the ID - #[text_signature = "(self, id)"] + #[pyo3(text_signature = "(self, id)")] fn id_to_token(&self, id: u32) -> Option { self.model.read().unwrap().id_to_token(id) } @@ -182,7 +182,7 @@ impl PyModel { /// /// Returns: /// :obj:`List[str]`: The list of saved files - #[text_signature = "(self, folder, prefix)"] + #[pyo3(text_signature = "(self, folder, prefix)")] fn save<'a>( &self, folder: &str, @@ -248,8 +248,10 @@ impl PyModel { /// /// fuse_unk (:obj:`bool`, `optional`): /// Whether to fuse any subsequent unknown tokens into a single one -#[pyclass(extends=PyModel, module = "tokenizers.models", name=BPE)] -#[text_signature = "(self, vocab=None, merges=None, cache_capacity=None, dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=None)"] +#[pyclass(extends=PyModel, module = "tokenizers.models", name = "BPE")] +#[pyo3( + text_signature = "(self, vocab=None, merges=None, cache_capacity=None, dropout=None, unk_token=None, continuing_subword_prefix=None, end_of_word_suffix=None, fuse_unk=None)" +)] pub struct PyBPE {} impl PyBPE { @@ -437,7 +439,7 @@ impl PyBPE { /// A :obj:`Tuple` with the vocab and the merges: /// The vocabulary and merges loaded into memory #[staticmethod] - #[text_signature = "(self, vocab, merges)"] + #[pyo3(text_signature = "(self, vocab, merges)")] fn read_file(vocab: &str, merges: &str) -> PyResult<(Vocab, Merges)> { BPE::read_file(vocab, merges).map_err(|e| { exceptions::PyException::new_err(format!( @@ -469,7 +471,7 @@ impl PyBPE { /// :class:`~tokenizers.models.BPE`: An instance of BPE loaded from these files #[classmethod] #[args(kwargs = "**")] - #[text_signature = "(cls, vocab, merge, **kwargs)"] + #[pyo3(text_signature = "(cls, vocab, merge, **kwargs)")] fn from_file( _cls: &PyType, py: Python, @@ -502,8 +504,8 @@ impl PyBPE { /// /// max_input_chars_per_word (:obj:`int`, `optional`): /// The maximum number of characters to authorize in a single word. -#[pyclass(extends=PyModel, module = "tokenizers.models", name=WordPiece)] -#[text_signature = "(self, vocab, unk_token, max_input_chars_per_word)"] +#[pyclass(extends=PyModel, module = "tokenizers.models", name = "WordPiece")] +#[pyo3(text_signature = "(self, vocab, unk_token, max_input_chars_per_word)")] pub struct PyWordPiece {} impl PyWordPiece { @@ -613,7 +615,7 @@ impl PyWordPiece { /// Returns: /// :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` #[staticmethod] - #[text_signature = "(vocab)"] + #[pyo3(text_signature = "(vocab)")] fn read_file(vocab: &str) -> PyResult { WordPiece::read_file(vocab).map_err(|e| { exceptions::PyException::new_err(format!("Error while reading WordPiece file: {}", e)) @@ -639,7 +641,7 @@ impl PyWordPiece { /// :class:`~tokenizers.models.WordPiece`: An instance of WordPiece loaded from file #[classmethod] #[args(kwargs = "**")] - #[text_signature = "(vocab, **kwargs)"] + #[pyo3(text_signature = "(vocab, **kwargs)")] fn from_file( _cls: &PyType, py: Python, @@ -663,8 +665,8 @@ impl PyWordPiece { /// /// unk_token (:obj:`str`, `optional`): /// The unknown token to be used by the model. -#[pyclass(extends=PyModel, module = "tokenizers.models", name=WordLevel)] -#[text_signature = "(self, vocab, unk_token)"] +#[pyclass(extends=PyModel, module = "tokenizers.models", name = "WordLevel")] +#[pyo3(text_signature = "(self, vocab, unk_token)")] pub struct PyWordLevel {} #[pymethods] @@ -725,7 +727,7 @@ impl PyWordLevel { /// Returns: /// :obj:`Dict[str, int]`: The vocabulary as a :obj:`dict` #[staticmethod] - #[text_signature = "(vocab)"] + #[pyo3(text_signature = "(vocab)")] fn read_file(vocab: &str) -> PyResult { WordLevel::read_file(vocab).map_err(|e| { exceptions::PyException::new_err(format!("Error while reading WordLevel file: {}", e)) @@ -751,7 +753,7 @@ impl PyWordLevel { /// :class:`~tokenizers.models.WordLevel`: An instance of WordLevel loaded from file #[classmethod] #[args(unk_token = "None")] - #[text_signature = "(vocab, unk_token)"] + #[pyo3(text_signature = "(vocab, unk_token)")] fn from_file( _cls: &PyType, py: Python, @@ -773,8 +775,8 @@ impl PyWordLevel { /// Args: /// vocab (:obj:`List[Tuple[str, float]]`, `optional`): /// A list of vocabulary items and their relative score [("am", -0.2442),...] -#[pyclass(extends=PyModel, module = "tokenizers.models", name=Unigram)] -#[text_signature = "(self, vocab)"] +#[pyclass(extends=PyModel, module = "tokenizers.models", name = "Unigram")] +#[pyo3(text_signature = "(self, vocab)")] pub struct PyUnigram {} #[pymethods] @@ -809,8 +811,8 @@ mod test { let py_bpe = py_model.get_as_subtype().unwrap(); let gil = Python::acquire_gil(); assert_eq!( - "tokenizers.models.BPE", - py_bpe.as_ref(gil.python()).get_type().name() + "BPE", + py_bpe.as_ref(gil.python()).get_type().name().unwrap() ); } diff --git a/bindings/python/src/normalizers.rs b/bindings/python/src/normalizers.rs index fb1a5cc29..994c62e92 100644 --- a/bindings/python/src/normalizers.rs +++ b/bindings/python/src/normalizers.rs @@ -3,7 +3,6 @@ use std::sync::{Arc, RwLock}; use pyo3::exceptions; use pyo3::prelude::*; use pyo3::types::*; -use pyo3::PySequenceProtocol; use crate::error::ToPyResult; use crate::utils::{PyNormalizedString, PyNormalizedStringRefMut, PyPattern}; @@ -43,7 +42,7 @@ impl PyNormalizedStringMut<'_> { /// /// This class is not supposed to be instantiated directly. Instead, any implementation of a /// Normalizer will return an instance of this class when instantiated. -#[pyclass(dict, module = "tokenizers.normalizers", name=Normalizer)] +#[pyclass(dict, module = "tokenizers.normalizers", name = "Normalizer", subclass)] #[derive(Clone, Serialize, Deserialize)] pub struct PyNormalizer { #[serde(flatten)] @@ -144,7 +143,7 @@ impl PyNormalizer { /// normalized (:class:`~tokenizers.NormalizedString`): /// The normalized string on which to apply this /// :class:`~tokenizers.normalizers.Normalizer` - #[text_signature = "(self, normalized)"] + #[pyo3(text_signature = "(self, normalized)")] fn normalize(&self, mut normalized: PyNormalizedStringMut) -> PyResult<()> { normalized.normalize_with(&self.normalizer) } @@ -162,7 +161,7 @@ impl PyNormalizer { /// /// Returns: /// :obj:`str`: A string after normalization - #[text_signature = "(self, sequence)"] + #[pyo3(text_signature = "(self, sequence)")] fn normalize_str(&self, sequence: &str) -> PyResult { let mut normalized = NormalizedString::from(sequence); ToPyResult(self.normalizer.normalize(&mut normalized)).into_py()?; @@ -217,8 +216,10 @@ macro_rules! setter { /// /// lowercase (:obj:`bool`, `optional`, defaults to :obj:`True`): /// Whether to lowercase. -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=BertNormalizer)] -#[text_signature = "(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "BertNormalizer")] +#[pyo3( + text_signature = "(self, clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True)" +)] pub struct PyBertNormalizer {} #[pymethods] impl PyBertNormalizer { @@ -287,8 +288,8 @@ impl PyBertNormalizer { } /// NFD Unicode Normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=NFD)] -#[text_signature = "(self)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFD")] +#[pyo3(text_signature = "(self)")] pub struct PyNFD {} #[pymethods] impl PyNFD { @@ -299,8 +300,8 @@ impl PyNFD { } /// NFKD Unicode Normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=NFKD)] -#[text_signature = "(self)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFKD")] +#[pyo3(text_signature = "(self)")] pub struct PyNFKD {} #[pymethods] impl PyNFKD { @@ -311,8 +312,8 @@ impl PyNFKD { } /// NFC Unicode Normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=NFC)] -#[text_signature = "(self)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFC")] +#[pyo3(text_signature = "(self)")] pub struct PyNFC {} #[pymethods] impl PyNFC { @@ -323,8 +324,8 @@ impl PyNFC { } /// NFKC Unicode Normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=NFKC)] -#[text_signature = "(self)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "NFKC")] +#[pyo3(text_signature = "(self)")] pub struct PyNFKC {} #[pymethods] impl PyNFKC { @@ -340,7 +341,7 @@ impl PyNFKC { /// Args: /// normalizers (:obj:`List[Normalizer]`): /// A list of Normalizer to be run as a sequence -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=Sequence)] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Sequence")] pub struct PySequence {} #[pymethods] impl PySequence { @@ -363,18 +364,15 @@ impl PySequence { fn __getnewargs__<'p>(&self, py: Python<'p>) -> &'p PyTuple { PyTuple::new(py, &[PyList::empty(py)]) } -} -#[pyproto] -impl PySequenceProtocol for PySequence { fn __len__(&self) -> usize { 0 } } /// Lowercase Normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=Lowercase)] -#[text_signature = "(self)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Lowercase")] +#[pyo3(text_signature = "(self)")] pub struct PyLowercase {} #[pymethods] impl PyLowercase { @@ -385,8 +383,8 @@ impl PyLowercase { } /// Strip normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=Strip)] -#[text_signature = "(self, left=True, right=True)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Strip")] +#[pyo3(text_signature = "(self, left=True, right=True)")] pub struct PyStrip {} #[pymethods] impl PyStrip { @@ -418,8 +416,8 @@ impl PyStrip { } /// StripAccents normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=StripAccents)] -#[text_signature = "(self)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "StripAccents")] +#[pyo3(text_signature = "(self)")] pub struct PyStripAccents {} #[pymethods] impl PyStripAccents { @@ -430,8 +428,8 @@ impl PyStripAccents { } /// Nmt normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=Nmt)] -#[text_signature = "(self)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Nmt")] +#[pyo3(text_signature = "(self)")] pub struct PyNmt {} #[pymethods] impl PyNmt { @@ -443,8 +441,8 @@ impl PyNmt { /// Precompiled normalizer /// Don't use manually it is used for compatiblity for SentencePiece. -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=Precompiled)] -#[text_signature = "(self, precompiled_charsmap)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Precompiled")] +#[pyo3(text_signature = "(self, precompiled_charsmap)")] pub struct PyPrecompiled {} #[pymethods] impl PyPrecompiled { @@ -466,8 +464,8 @@ impl PyPrecompiled { } /// Replace normalizer -#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name=Replace)] -#[text_signature = "(self, pattern, content)"] +#[pyclass(extends=PyNormalizer, module = "tokenizers.normalizers", name = "Replace")] +#[pyo3(text_signature = "(self, pattern, content)")] pub struct PyReplace {} #[pymethods] impl PyReplace { @@ -630,8 +628,8 @@ mod test { let py_nfc = py_norm.get_as_subtype().unwrap(); let gil = Python::acquire_gil(); assert_eq!( - "tokenizers.normalizers.NFC", - py_nfc.as_ref(gil.python()).get_type().name() + "NFC", + py_nfc.as_ref(gil.python()).get_type().name().unwrap() ); } diff --git a/bindings/python/src/pre_tokenizers.rs b/bindings/python/src/pre_tokenizers.rs index f858ecf75..661b23d89 100644 --- a/bindings/python/src/pre_tokenizers.rs +++ b/bindings/python/src/pre_tokenizers.rs @@ -28,7 +28,12 @@ use super::utils::*; /// /// This class is not supposed to be instantiated directly. Instead, any implementation of a /// PreTokenizer will return an instance of this class when instantiated. -#[pyclass(dict, module = "tokenizers.pre_tokenizers", name=PreTokenizer)] +#[pyclass( + dict, + module = "tokenizers.pre_tokenizers", + name = "PreTokenizer", + subclass +)] #[derive(Clone, Serialize, Deserialize)] pub struct PyPreTokenizer { #[serde(flatten)] @@ -146,7 +151,7 @@ impl PyPreTokenizer { /// pretok (:class:`~tokenizers.PreTokenizedString): /// The pre-tokenized string on which to apply this /// :class:`~tokenizers.pre_tokenizers.PreTokenizer` - #[text_signature = "(self, pretok)"] + #[pyo3(text_signature = "(self, pretok)")] fn pre_tokenize(&self, pretok: &mut PyPreTokenizedString) -> PyResult<()> { ToPyResult(self.pretok.pre_tokenize(&mut pretok.pretok)).into() } @@ -166,7 +171,7 @@ impl PyPreTokenizer { /// Returns: /// :obj:`List[Tuple[str, Offsets]]`: /// A list of tuple with the pre-tokenized parts and their offsets - #[text_signature = "(self, sequence)"] + #[pyo3(text_signature = "(self, sequence)")] fn pre_tokenize_str(&self, s: &str) -> PyResult> { let mut pretokenized = tk::tokenizer::PreTokenizedString::from(s); @@ -231,8 +236,8 @@ macro_rules! setter { /// use_regex (:obj:`bool`, `optional`, defaults to :obj:`True`): /// Set this to :obj:`False` to prevent this `pre_tokenizer` from using /// the GPT2 specific regexp for spliting on whitespace. -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=ByteLevel)] -#[text_signature = "(self, add_prefix_space=True, use_regex=True)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "ByteLevel")] +#[pyo3(text_signature = "(self, add_prefix_space=True, use_regex=True)")] pub struct PyByteLevel {} #[pymethods] impl PyByteLevel { @@ -281,7 +286,7 @@ impl PyByteLevel { /// Returns: /// :obj:`List[str]`: A list of characters that compose the alphabet #[staticmethod] - #[text_signature = "()"] + #[pyo3(text_signature = "()")] fn alphabet() -> Vec { ByteLevel::alphabet() .into_iter() @@ -291,8 +296,8 @@ impl PyByteLevel { } /// This pre-tokenizer simply splits using the following regex: `\w+|[^\w\s]+` -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=Whitespace)] -#[text_signature = "(self)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Whitespace")] +#[pyo3(text_signature = "(self)")] pub struct PyWhitespace {} #[pymethods] impl PyWhitespace { @@ -303,8 +308,8 @@ impl PyWhitespace { } /// This pre-tokenizer simply splits on the whitespace. Works like `.split()` -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=WhitespaceSplit)] -#[text_signature = "(self)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "WhitespaceSplit")] +#[pyo3(text_signature = "(self)")] pub struct PyWhitespaceSplit {} #[pymethods] impl PyWhitespaceSplit { @@ -331,8 +336,8 @@ impl PyWhitespaceSplit { /// /// invert (:obj:`bool`, `optional`, defaults to :obj:`False`): /// Whether to invert the pattern. -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=Split)] -#[text_signature = "(self, pattern, behavior, invert=False)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Split")] +#[pyo3(text_signature = "(self, pattern, behavior, invert=False)")] pub struct PySplit {} #[pymethods] impl PySplit { @@ -361,7 +366,7 @@ impl PySplit { /// Args: /// delimiter: str: /// The delimiter char that will be used to split input -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=CharDelimiterSplit)] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "CharDelimiterSplit")] pub struct PyCharDelimiterSplit {} #[pymethods] impl PyCharDelimiterSplit { @@ -392,8 +397,8 @@ impl PyCharDelimiterSplit { /// /// This pre-tokenizer splits tokens on spaces, and also on punctuation. /// Each occurence of a punctuation character will be treated separately. -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=BertPreTokenizer)] -#[text_signature = "(self)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "BertPreTokenizer")] +#[pyo3(text_signature = "(self)")] pub struct PyBertPreTokenizer {} #[pymethods] impl PyBertPreTokenizer { @@ -410,8 +415,8 @@ impl PyBertPreTokenizer { /// The behavior to use when splitting. /// Choices: "removed", "isolated" (default), "merged_with_previous", "merged_with_next", /// "contiguous" -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=Punctuation)] -#[text_signature = "(self, behavior=\"isolated\")"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Punctuation")] +#[pyo3(text_signature = "(self, behavior=\"isolated\")")] pub struct PyPunctuation {} #[pymethods] impl PyPunctuation { @@ -423,8 +428,8 @@ impl PyPunctuation { } /// This pre-tokenizer composes other pre_tokenizers and applies them in sequence -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=Sequence)] -#[text_signature = "(self, pretokenizers)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Sequence")] +#[pyo3(text_signature = "(self, pretokenizers)")] pub struct PySequence {} #[pymethods] impl PySequence { @@ -464,8 +469,8 @@ impl PySequence { /// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): /// Whether to add a space to the first word if there isn't already one. This /// lets us treat `hello` exactly like `say hello`. -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=Metaspace)] -#[text_signature = "(self, replacement=\"_\", add_prefix_space=True)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Metaspace")] +#[pyo3(text_signature = "(self, replacement=\"_\", add_prefix_space=True)")] pub struct PyMetaspace {} #[pymethods] impl PyMetaspace { @@ -514,8 +519,8 @@ impl PyMetaspace { /// If set to False, digits will grouped as follows:: /// /// "Call 123 please" -> "Call ", "123", " please" -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=Digits)] -#[text_signature = "(self, individual_digits=False)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "Digits")] +#[pyo3(text_signature = "(self, individual_digits=False)")] pub struct PyDigits {} #[pymethods] impl PyDigits { @@ -540,8 +545,8 @@ impl PyDigits { /// It roughly follows https://github.com/google/sentencepiece/blob/master/data/Scripts.txt /// Actually Hiragana and Katakana are fused with Han, and 0x30FC is Han too. /// This mimicks SentencePiece Unigram implementation. -#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name=UnicodeScripts)] -#[text_signature = "(self)"] +#[pyclass(extends=PyPreTokenizer, module = "tokenizers.pre_tokenizers", name = "UnicodeScripts")] +#[pyo3(text_signature = "(self)")] pub struct PyUnicodeScripts {} #[pymethods] impl PyUnicodeScripts { @@ -704,8 +709,8 @@ mod test { let py_wsp = py_norm.get_as_subtype().unwrap(); let gil = Python::acquire_gil(); assert_eq!( - "tokenizers.pre_tokenizers.Whitespace", - py_wsp.as_ref(gil.python()).get_type().name() + "Whitespace", + py_wsp.as_ref(gil.python()).get_type().name().unwrap() ); } diff --git a/bindings/python/src/processors.rs b/bindings/python/src/processors.rs index 12990b3d4..4e346f7d6 100644 --- a/bindings/python/src/processors.rs +++ b/bindings/python/src/processors.rs @@ -20,7 +20,12 @@ use tokenizers as tk; /// /// This class is not supposed to be instantiated directly. Instead, any implementation of /// a PostProcessor will return an instance of this class when instantiated. -#[pyclass(dict, module = "tokenizers.processors", name=PostProcessor)] +#[pyclass( + dict, + module = "tokenizers.processors", + name = "PostProcessor", + subclass +)] #[derive(Clone, Deserialize, Serialize)] pub struct PyPostProcessor { #[serde(flatten)] @@ -100,7 +105,7 @@ impl PyPostProcessor { /// /// Returns: /// :obj:`int`: The number of tokens to add - #[text_signature = "(self, is_pair)"] + #[pyo3(text_signature = "(self, is_pair)")] fn num_special_tokens_to_add(&self, is_pair: bool) -> usize { self.processor.added_tokens(is_pair) } @@ -120,7 +125,7 @@ impl PyPostProcessor { /// Return: /// :class:`~tokenizers.Encoding`: The final encoding #[args(pair = "None", add_special_tokens = "true")] - #[text_signature = "(self, encoding, pair=None, add_special_tokens=True)"] + #[pyo3(text_signature = "(self, encoding, pair=None, add_special_tokens=True)")] fn process( &self, encoding: &PyEncoding, @@ -149,8 +154,8 @@ impl PyPostProcessor { /// /// cls (:obj:`Tuple[str, int]`): /// A tuple with the string representation of the CLS token, and its id -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name=BertProcessing)] -#[text_signature = "(self, sep, cls)"] +#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "BertProcessing")] +#[pyo3(text_signature = "(self, sep, cls)")] pub struct PyBertProcessing {} #[pymethods] impl PyBertProcessing { @@ -191,8 +196,8 @@ impl PyBertProcessing { /// add_prefix_space (:obj:`bool`, `optional`, defaults to :obj:`True`): /// Whether the add_prefix_space option was enabled during pre-tokenization. This /// is relevant because it defines the way the offsets are trimmed out. -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name=RobertaProcessing)] -#[text_signature = "(self, sep, cls, trim_offsets=True, add_prefix_space=True)"] +#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "RobertaProcessing")] +#[pyo3(text_signature = "(self, sep, cls, trim_offsets=True, add_prefix_space=True)")] pub struct PyRobertaProcessing {} #[pymethods] impl PyRobertaProcessing { @@ -226,8 +231,8 @@ impl PyRobertaProcessing { /// Args: /// trim_offsets (:obj:`bool`): /// Whether to trim the whitespaces from the produced offsets. -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name=ByteLevel)] -#[text_signature = "(self, trim_offsets=True)"] +#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "ByteLevel")] +#[pyo3(text_signature = "(self, trim_offsets=True)")] pub struct PyByteLevel {} #[pymethods] impl PyByteLevel { @@ -378,8 +383,8 @@ impl FromPyObject<'_> for PyTemplate { /// /// The given dict expects the provided :obj:`ids` and :obj:`tokens` lists to have /// the same length. -#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name=TemplateProcessing)] -#[text_signature = "(self, single, pair, special_tokens)"] +#[pyclass(extends=PyPostProcessor, module = "tokenizers.processors", name = "TemplateProcessing")] +#[pyo3(text_signature = "(self, single, pair, special_tokens)")] pub struct PyTemplateProcessing {} #[pymethods] impl PyTemplateProcessing { @@ -428,8 +433,8 @@ mod test { let py_bert = py_proc.get_as_subtype().unwrap(); let gil = Python::acquire_gil(); assert_eq!( - "tokenizers.processors.BertProcessing", - py_bert.as_ref(gil.python()).get_type().name() + "BertProcessing", + py_bert.as_ref(gil.python()).get_type().name().unwrap() ); } diff --git a/bindings/python/src/token.rs b/bindings/python/src/token.rs index eb2a47286..f1db9974d 100644 --- a/bindings/python/src/token.rs +++ b/bindings/python/src/token.rs @@ -1,7 +1,7 @@ use pyo3::prelude::*; use tk::Token; -#[pyclass(module = "tokenizers", name=Token)] +#[pyclass(module = "tokenizers", name = "Token")] #[derive(Clone)] pub struct PyToken { token: Token, diff --git a/bindings/python/src/tokenizer.rs b/bindings/python/src/tokenizer.rs index 89073a4fa..e35f96c5a 100644 --- a/bindings/python/src/tokenizer.rs +++ b/bindings/python/src/tokenizer.rs @@ -1,12 +1,12 @@ use std::collections::{hash_map::DefaultHasher, HashMap}; use std::hash::{Hash, Hasher}; -use numpy::PyArray1; +use numpy::{npyffi, PyArray1}; use pyo3::class::basic::CompareOp; use pyo3::exceptions; use pyo3::prelude::*; use pyo3::types::*; -use pyo3::PyObjectProtocol; +use pyo3::AsPyPointer; use tk::models::bpe::BPE; use tk::tokenizer::{ Model, PaddingDirection, PaddingParams, PaddingStrategy, PostProcessor, TokenizerImpl, @@ -55,8 +55,10 @@ use crate::utils::{MaybeSizedIterator, PyBufferedIterator}; /// lowercasing the text, the token could be extract from the input ``"I saw a lion /// Yesterday"``. /// -#[pyclass(dict, module = "tokenizers", name=AddedToken)] -#[text_signature = "(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True)"] +#[pyclass(dict, module = "tokenizers", name = "AddedToken")] +#[pyo3( + text_signature = "(self, content, single_word=False, lstrip=False, rstrip=False, normalized=True)" +)] pub struct PyAddedToken { pub content: String, pub is_special_token: bool, @@ -199,10 +201,8 @@ impl PyAddedToken { fn get_normalized(&self) -> bool { self.get_token().normalized } -} -#[pyproto] -impl PyObjectProtocol for PyAddedToken { - fn __str__(&'p self) -> PyResult<&'p str> { + + fn __str__(&self) -> PyResult<&str> { Ok(&self.content) } @@ -259,38 +259,54 @@ impl<'s> From> for tk::InputSequence<'s> { struct PyArrayUnicode(Vec); impl FromPyObject<'_> for PyArrayUnicode { fn extract(ob: &PyAny) -> PyResult { - let array = ob.downcast::>()?; - let arr = array.as_array_ptr(); - let (type_num, elsize, alignment, data) = unsafe { + // SAFETY Making sure the pointer is a valid numpy array requires calling numpy C code + if unsafe { npyffi::PyArray_Check(ob.py(), ob.as_ptr()) } == 0 { + return Err(exceptions::PyTypeError::new_err("Expected an np.array")); + } + let arr = ob.as_ptr() as *mut npyffi::PyArrayObject; + // SAFETY Getting all the metadata about the numpy array to check its sanity + let (type_num, elsize, alignment, data, nd, flags) = unsafe { let desc = (*arr).descr; ( (*desc).type_num, (*desc).elsize as usize, (*desc).alignment as usize, (*arr).data, + (*arr).nd, + (*arr).flags, ) }; - let n_elem = array.shape()[0]; - // type_num == 19 => Unicode - if type_num != 19 { + if nd != 1 { + return Err(exceptions::PyTypeError::new_err( + "Expected a 1 dimensional np.array", + )); + } + if flags & (npyffi::NPY_ARRAY_C_CONTIGUOUS | npyffi::NPY_ARRAY_F_CONTIGUOUS) == 0 { + return Err(exceptions::PyTypeError::new_err( + "Expected a contiguous np.array", + )); + } + if type_num != npyffi::types::NPY_TYPES::NPY_UNICODE as i32 { return Err(exceptions::PyTypeError::new_err( "Expected a np.array[dtype='U']", )); } + // SAFETY Looking at the raw numpy data to create new owned Rust strings via copies (so it's safe afterwards). unsafe { + let n_elem = *(*arr).dimensions as usize; let all_bytes = std::slice::from_raw_parts(data as *const u8, elsize * n_elem); let seq = (0..n_elem) .map(|i| { let bytes = &all_bytes[i * elsize..(i + 1) * elsize]; - let unicode = pyo3::ffi::PyUnicode_FromUnicode( + let unicode = pyo3::ffi::PyUnicode_FromKindAndData( + pyo3::ffi::PyUnicode_4BYTE_KIND as _, bytes.as_ptr() as *const _, elsize as isize / alignment as isize, ); - let gil = Python::acquire_gil(); - let py = gil.python(); + let py = ob.py(); let obj = PyObject::from_owned_ptr(py, unicode); let s = obj.cast_as::(py)?; Ok(s.to_string_lossy().trim_matches(char::from(0)).to_owned()) @@ -310,32 +326,18 @@ impl From for tk::InputSequence<'_> { struct PyArrayStr(Vec); impl FromPyObject<'_> for PyArrayStr { fn extract(ob: &PyAny) -> PyResult { - let array = ob.downcast::>()?; - let arr = array.as_array_ptr(); - let (type_num, data) = unsafe { ((*(*arr).descr).type_num, (*arr).data) }; - let n_elem = array.shape()[0]; - - if type_num != 17 { - return Err(exceptions::PyTypeError::new_err( - "Expected a np.array[dtype='O']", - )); - } - - unsafe { - let objects = std::slice::from_raw_parts(data as *const PyObject, n_elem); - - let seq = objects - .iter() - .map(|obj| { - let gil = Python::acquire_gil(); - let py = gil.python(); - let s = obj.cast_as::(py)?; - Ok(s.to_string_lossy().into_owned()) - }) - .collect::>>()?; + let array = ob.downcast::>()?; + let seq = array + .readonly() + .as_array() + .iter() + .map(|obj| { + let s = obj.cast_as::(ob.py())?; + Ok(s.to_string_lossy().into_owned()) + }) + .collect::>>()?; - Ok(Self(seq)) - } + Ok(Self(seq)) } } impl From for tk::InputSequence<'_> { @@ -438,8 +440,8 @@ type Tokenizer = TokenizerImpl PyResult { let tokenizer: PyResult<_> = ToPyResult(json.parse()).into(); Ok(Self::new(tokenizer?)) @@ -518,7 +520,7 @@ impl PyTokenizer { /// Returns: /// :class:`~tokenizers.Tokenizer`: The new tokenizer #[staticmethod] - #[text_signature = "(path)"] + #[pyo3(text_signature = "(path)")] fn from_file(path: &str) -> PyResult { let tokenizer: PyResult<_> = ToPyResult(Tokenizer::from_file(path)).into(); Ok(Self::new(tokenizer?)) @@ -533,7 +535,7 @@ impl PyTokenizer { /// Returns: /// :class:`~tokenizers.Tokenizer`: The new tokenizer #[staticmethod] - #[text_signature = "(buffer)"] + #[pyo3(text_signature = "(buffer)")] fn from_buffer(buffer: &PyBytes) -> PyResult { let tokenizer = serde_json::from_slice(buffer.as_bytes()).map_err(|e| { exceptions::PyValueError::new_err(format!( @@ -561,7 +563,7 @@ impl PyTokenizer { /// :class:`~tokenizers.Tokenizer`: The new tokenizer #[staticmethod] #[args(revision = "String::from(\"main\")", auth_token = "None")] - #[text_signature = "(identifier, revision=\"main\", auth_token=None)"] + #[pyo3(text_signature = "(identifier, revision=\"main\", auth_token=None)")] fn from_pretrained( identifier: &str, revision: String, @@ -590,7 +592,7 @@ impl PyTokenizer { /// Returns: /// :obj:`str`: A string representing the serialized Tokenizer #[args(pretty = false)] - #[text_signature = "(self, pretty=False)"] + #[pyo3(text_signature = "(self, pretty=False)")] fn to_str(&self, pretty: bool) -> PyResult { ToPyResult(self.tokenizer.to_string(pretty)).into() } @@ -604,7 +606,7 @@ impl PyTokenizer { /// pretty (:obj:`bool`, defaults to :obj:`True`): /// Whether the JSON file should be pretty formatted. #[args(pretty = true)] - #[text_signature = "(self, path, pretty=True)"] + #[pyo3(text_signature = "(self, path, pretty=True)")] fn save(&self, path: &str, pretty: bool) -> PyResult<()> { ToPyResult(self.tokenizer.save(path, pretty)).into() } @@ -612,7 +614,7 @@ impl PyTokenizer { /// Return the number of special tokens that would be added for single/pair sentences. /// :param is_pair: Boolean indicating if the input would be a single sentence or a pair /// :return: - #[text_signature = "(self, is_pair)"] + #[pyo3(text_signature = "(self, is_pair)")] fn num_special_tokens_to_add(&self, is_pair: bool) -> usize { self.tokenizer .get_post_processor() @@ -628,7 +630,7 @@ impl PyTokenizer { /// Returns: /// :obj:`Dict[str, int]`: The vocabulary #[args(with_added_tokens = true)] - #[text_signature = "(self, with_added_tokens=True)"] + #[pyo3(text_signature = "(self, with_added_tokens=True)")] fn get_vocab(&self, with_added_tokens: bool) -> HashMap { self.tokenizer.get_vocab(with_added_tokens) } @@ -642,7 +644,7 @@ impl PyTokenizer { /// Returns: /// :obj:`int`: The size of the vocabulary #[args(with_added_tokens = true)] - #[text_signature = "(self, with_added_tokens=True)"] + #[pyo3(text_signature = "(self, with_added_tokens=True)")] fn get_vocab_size(&self, with_added_tokens: bool) -> usize { self.tokenizer.get_vocab_size(with_added_tokens) } @@ -664,7 +666,9 @@ impl PyTokenizer { /// direction (:obj:`str`, defaults to :obj:`right`): /// Truncate direction #[args(kwargs = "**")] - #[text_signature = "(self, max_length, stride=0, strategy='longest_first', direction='right')"] + #[pyo3( + text_signature = "(self, max_length, stride=0, strategy='longest_first', direction='right')" + )] fn enable_truncation(&mut self, max_length: usize, kwargs: Option<&PyDict>) -> PyResult<()> { let mut params = TruncationParams { max_length, @@ -714,7 +718,7 @@ impl PyTokenizer { } /// Disable truncation - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn no_truncation(&mut self) { self.tokenizer.with_truncation(None); } @@ -764,7 +768,9 @@ impl PyTokenizer { /// If specified, the length at which to pad. If not specified we pad using the size of /// the longest sequence in a batch. #[args(kwargs = "**")] - #[text_signature = "(self, direction='right', pad_id=0, pad_type_id=0, pad_token='[PAD]', length=None, pad_to_multiple_of=None)"] + #[pyo3( + text_signature = "(self, direction='right', pad_id=0, pad_type_id=0, pad_token='[PAD]', length=None, pad_to_multiple_of=None)" + )] fn enable_padding(&mut self, kwargs: Option<&PyDict>) -> PyResult<()> { let mut params = PaddingParams::default(); @@ -822,7 +828,7 @@ impl PyTokenizer { } /// Disable padding - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn no_padding(&mut self) { self.tokenizer.with_padding(None); } @@ -891,7 +897,9 @@ impl PyTokenizer { /// :class:`~tokenizers.Encoding`: The encoded result /// #[args(pair = "None", is_pretokenized = "false", add_special_tokens = "true")] - #[text_signature = "(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True)"] + #[pyo3( + text_signature = "(self, sequence, pair=None, is_pretokenized=False, add_special_tokens=True)" + )] fn encode( &self, sequence: &PyAny, @@ -956,7 +964,7 @@ impl PyTokenizer { /// A :obj:`List` of :class:`~tokenizers.Encoding`: The encoded batch /// #[args(is_pretokenized = "false", add_special_tokens = "true")] - #[text_signature = "(self, input, is_pretokenized=False, add_special_tokens=True)"] + #[pyo3(text_signature = "(self, input, is_pretokenized=False, add_special_tokens=True)")] fn encode_batch( &self, input: Vec<&PyAny>, @@ -999,7 +1007,7 @@ impl PyTokenizer { /// Returns: /// :obj:`str`: The decoded string #[args(skip_special_tokens = true)] - #[text_signature = "(self, ids, skip_special_tokens=True)"] + #[pyo3(text_signature = "(self, ids, skip_special_tokens=True)")] fn decode(&self, ids: Vec, skip_special_tokens: bool) -> PyResult { ToPyResult(self.tokenizer.decode(ids, skip_special_tokens)).into() } @@ -1016,7 +1024,7 @@ impl PyTokenizer { /// Returns: /// :obj:`List[str]`: A list of decoded strings #[args(skip_special_tokens = true)] - #[text_signature = "(self, sequences, skip_special_tokens=True)"] + #[pyo3(text_signature = "(self, sequences, skip_special_tokens=True)")] fn decode_batch( &self, sequences: Vec>, @@ -1036,7 +1044,7 @@ impl PyTokenizer { /// /// Returns: /// :obj:`Optional[int]`: An optional id, :obj:`None` if out of vocabulary - #[text_signature = "(self, token)"] + #[pyo3(text_signature = "(self, token)")] fn token_to_id(&self, token: &str) -> Option { self.tokenizer.token_to_id(token) } @@ -1049,7 +1057,7 @@ impl PyTokenizer { /// /// Returns: /// :obj:`Optional[str]`: An optional token, :obj:`None` if out of vocabulary - #[text_signature = "(self, id)"] + #[pyo3(text_signature = "(self, id)")] fn id_to_token(&self, id: u32) -> Option { self.tokenizer.id_to_token(id) } @@ -1066,7 +1074,7 @@ impl PyTokenizer { /// /// Returns: /// :obj:`int`: The number of tokens that were created in the vocabulary - #[text_signature = "(self, tokens)"] + #[pyo3(text_signature = "(self, tokens)")] fn add_tokens(&mut self, tokens: &PyList) -> PyResult { let tokens = tokens .into_iter() @@ -1103,7 +1111,7 @@ impl PyTokenizer { /// /// Returns: /// :obj:`int`: The number of tokens that were created in the vocabulary - #[text_signature = "(self, tokens)"] + #[pyo3(text_signature = "(self, tokens)")] fn add_special_tokens(&mut self, tokens: &PyList) -> PyResult { let tokens = tokens .into_iter() @@ -1137,7 +1145,7 @@ impl PyTokenizer { /// trainer (:obj:`~tokenizers.trainers.Trainer`, `optional`): /// An optional trainer that should be used to train our Model #[args(trainer = "None")] - #[text_signature = "(self, files, trainer = None)"] + #[pyo3(text_signature = "(self, files, trainer = None)")] fn train(&mut self, files: Vec, trainer: Option<&mut PyTrainer>) -> PyResult<()> { let mut trainer = trainer.map_or_else(|| self.tokenizer.get_model().get_trainer(), |t| t.clone()); @@ -1173,7 +1181,7 @@ impl PyTokenizer { /// The total number of sequences in the iterator. This is used to /// provide meaningful progress tracking #[args(trainer = "None", length = "None")] - #[text_signature = "(self, iterator, trainer=None, length=None)"] + #[pyo3(text_signature = "(self, iterator, trainer=None, length=None)")] fn train_from_iterator( &mut self, py: Python, @@ -1239,7 +1247,7 @@ impl PyTokenizer { /// Returns: /// :class:`~tokenizers.Encoding`: The final post-processed encoding #[args(pair = "None", add_special_tokens = true)] - #[text_signature = "(self, encoding, pair=None, add_special_tokens=True)"] + #[pyo3(text_signature = "(self, encoding, pair=None, add_special_tokens=True)")] fn post_process( &self, encoding: &PyEncoding, diff --git a/bindings/python/src/trainers.rs b/bindings/python/src/trainers.rs index d611a595b..e03700d38 100644 --- a/bindings/python/src/trainers.rs +++ b/bindings/python/src/trainers.rs @@ -15,7 +15,7 @@ use tokenizers as tk; /// /// This class is not supposed to be instantiated directly. Instead, any implementation of a /// Trainer will return an instance of this class when instantiated. -#[pyclass(name=Trainer, module = "tokenizers.trainers", name=Trainer)] +#[pyclass(module = "tokenizers.trainers", name = "Trainer", subclass)] #[derive(Clone, Deserialize, Serialize)] pub struct PyTrainer { #[serde(flatten)] @@ -164,7 +164,7 @@ macro_rules! setter { /// /// end_of_word_suffix (:obj:`str`, `optional`): /// A suffix to be used for every subword that is a end-of-word. -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name=BpeTrainer)] +#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "BpeTrainer")] pub struct PyBpeTrainer {} #[pymethods] impl PyBpeTrainer { @@ -367,8 +367,10 @@ impl PyBpeTrainer { /// /// end_of_word_suffix (:obj:`str`, `optional`): /// A suffix to be used for every subword that is a end-of-word. -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name=WordPieceTrainer)] -#[text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[], limit_alphabet=None, initial_alphabet= [],continuing_subword_prefix=\"##\", end_of_word_suffix=None)"] +#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "WordPieceTrainer")] +#[pyo3( + text_signature = "(self, vocab_size=30000, min_frequency=0, show_progress=True, special_tokens=[], limit_alphabet=None, initial_alphabet= [],continuing_subword_prefix=\"##\", end_of_word_suffix=None)" +)] pub struct PyWordPieceTrainer {} #[pymethods] impl PyWordPieceTrainer { @@ -557,7 +559,7 @@ impl PyWordPieceTrainer { /// /// special_tokens (:obj:`List[Union[str, AddedToken]]`): /// A list of special tokens the model should know of. -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name=WordLevelTrainer)] +#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "WordLevelTrainer")] pub struct PyWordLevelTrainer {} #[pymethods] impl PyWordLevelTrainer { @@ -713,8 +715,10 @@ impl PyWordLevelTrainer { /// n_sub_iterations (:obj:`int`): /// The number of iterations of the EM algorithm to perform before /// pruning the vocabulary. -#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name=UnigramTrainer)] -#[text_signature = "(self, vocab_size=8000, show_progress=True, special_tokens=[], shrinking_factor=0.75, unk_token=None, max_piece_length=16, n_sub_iterations=2)"] +#[pyclass(extends=PyTrainer, module = "tokenizers.trainers", name = "UnigramTrainer")] +#[pyo3( + text_signature = "(self, vocab_size=8000, show_progress=True, special_tokens=[], shrinking_factor=0.75, unk_token=None, max_piece_length=16, n_sub_iterations=2)" +)] pub struct PyUnigramTrainer {} #[pymethods] impl PyUnigramTrainer { @@ -864,8 +868,8 @@ mod tests { let py_bpe = py_trainer.get_as_subtype().unwrap(); let gil = Python::acquire_gil(); assert_eq!( - "tokenizers.trainers.BpeTrainer", - py_bpe.as_ref(gil.python()).get_type().name() + "BpeTrainer", + py_bpe.as_ref(gil.python()).get_type().name().unwrap() ); } } diff --git a/bindings/python/src/utils/iterators.rs b/bindings/python/src/utils/iterators.rs index 0715df512..cf6310b4d 100644 --- a/bindings/python/src/utils/iterators.rs +++ b/bindings/python/src/utils/iterators.rs @@ -1,5 +1,5 @@ use pyo3::prelude::*; -use pyo3::{AsPyPointer, PyNativeType}; +use pyo3::AsPyPointer; use std::collections::VecDeque; /// An simple iterator that can be instantiated with a specified length. diff --git a/bindings/python/src/utils/normalization.rs b/bindings/python/src/utils/normalization.rs index 39b1b7394..4bcd50fb6 100644 --- a/bindings/python/src/utils/normalization.rs +++ b/bindings/python/src/utils/normalization.rs @@ -4,7 +4,6 @@ use crate::error::ToPyResult; use pyo3::exceptions; use pyo3::prelude::*; use pyo3::types::*; -use pyo3::{PyMappingProtocol, PyObjectProtocol}; use tk::normalizer::{char_to_bytes, NormalizedString, Range, SplitDelimiterBehavior}; use tk::pattern::Pattern; @@ -192,7 +191,7 @@ fn slice( /// Args: /// sequence: str: /// The string sequence used to initialize this NormalizedString -#[pyclass(module = "tokenizers", name=NormalizedString)] +#[pyclass(module = "tokenizers", name = "NormalizedString")] #[derive(Clone)] pub struct PyNormalizedString { pub(crate) normalized: NormalizedString, @@ -217,91 +216,91 @@ impl PyNormalizedString { } /// Runs the NFD normalization - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn nfd(&mut self) { self.normalized.nfd(); } /// Runs the NFKD normalization - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn nfkd(&mut self) { self.normalized.nfkd(); } /// Runs the NFC normalization - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn nfc(&mut self) { self.normalized.nfc(); } /// Runs the NFKC normalization - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn nfkc(&mut self) { self.normalized.nfkc(); } /// Lowercase the string - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn lowercase(&mut self) { self.normalized.lowercase(); } /// Uppercase the string - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn uppercase(&mut self) { self.normalized.uppercase(); } /// Prepend the given sequence to the string - #[text_signature = "(self, s)"] + #[pyo3(text_signature = "(self, s)")] fn prepend(&mut self, s: &str) { self.normalized.prepend(s); } /// Append the given sequence to the string - #[text_signature = "(self, s)"] + #[pyo3(text_signature = "(self, s)")] fn append(&mut self, s: &str) { self.normalized.append(s); } /// Strip the left of the string - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn lstrip(&mut self) { self.normalized.lstrip(); } /// Strip the right of the string - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn rstrip(&mut self) { self.normalized.rstrip(); } /// Strip both ends of the string - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn strip(&mut self) { self.normalized.strip(); } /// Clears the string - #[text_signature = "(self)"] + #[pyo3(text_signature = "(self)")] fn clear(&mut self) { self.normalized.clear(); } /// Slice the string using the given range - #[text_signature = "(self, range)"] + #[pyo3(text_signature = "(self, range)")] fn slice(&self, range: PyRange) -> PyResult> { slice(&self.normalized, &range) } /// Filter each character of the string using the given func - #[text_signature = "(self, func)"] + #[pyo3(text_signature = "(self, func)")] fn filter(&mut self, func: &PyAny) -> PyResult<()> { filter(&mut self.normalized, func) } /// Calls the given function for each character of the string - #[text_signature = "(self, func)"] + #[pyo3(text_signature = "(self, func)")] fn for_each(&self, func: &PyAny) -> PyResult<()> { for_each(&self.normalized, func) } @@ -310,7 +309,7 @@ impl PyNormalizedString { /// /// Replaces each character of the string using the returned value. Each /// returned value **must** be a str of length 1 (ie a character). - #[text_signature = "(self, func)"] + #[pyo3(text_signature = "(self, func)")] fn map(&mut self, func: &PyAny) -> PyResult<()> { map(&mut self.normalized, func) } @@ -328,7 +327,7 @@ impl PyNormalizedString { /// /// Returns: /// A list of NormalizedString, representing each split - #[text_signature = "(self, pattern, behavior)"] + #[pyo3(text_signature = "(self, pattern, behavior)")] fn split( &mut self, pattern: PyPattern, @@ -349,14 +348,11 @@ impl PyNormalizedString { /// /// content: str: /// The content to be used as replacement - #[text_signature = "(self, pattern, content)"] + #[pyo3(text_signature = "(self, pattern, content)")] fn replace(&mut self, pattern: PyPattern, content: &str) -> PyResult<()> { ToPyResult(self.normalized.replace(pattern, content)).into() } -} -#[pyproto] -impl PyObjectProtocol<'p> for PyNormalizedString { fn __repr__(&self) -> String { format!( r#"NormalizedString(original="{}", normalized="{}")"#, @@ -365,14 +361,11 @@ impl PyObjectProtocol<'p> for PyNormalizedString { ) } - fn __str__(&'p self) -> &'p str { + fn __str__(&self) -> &str { self.normalized.get() } -} -#[pyproto] -impl PyMappingProtocol<'p> for PyNormalizedString { - fn __getitem__(&self, range: PyRange<'p>) -> PyResult> { + fn __getitem__(&self, range: PyRange<'_>) -> PyResult> { slice(&self.normalized, &range) } } @@ -389,7 +382,7 @@ impl From for NormalizedString { } } -#[pyclass(module = "tokenizers", name=NormalizedStringRefMut)] +#[pyclass(module = "tokenizers", name = "NormalizedStringRefMut")] #[derive(Clone)] pub struct PyNormalizedStringRefMut { inner: RefMutContainer, diff --git a/bindings/python/src/utils/pretokenization.rs b/bindings/python/src/utils/pretokenization.rs index b4d5a66aa..fb692c77f 100644 --- a/bindings/python/src/utils/pretokenization.rs +++ b/bindings/python/src/utils/pretokenization.rs @@ -147,8 +147,8 @@ fn to_encoding( /// Args: /// sequence: str: /// The string sequence used to initialize this PreTokenizedString -#[pyclass(module = "tokenizers", name=PreTokenizedString)] -#[text_signature = "(self, sequence)"] +#[pyclass(module = "tokenizers", name = "PreTokenizedString")] +#[pyo3(text_signature = "(self, sequence)")] pub struct PyPreTokenizedString { pub(crate) pretok: tk::PreTokenizedString, } @@ -182,7 +182,7 @@ impl PyPreTokenizedString { /// just return it directly. /// In order for the offsets to be tracked accurately, any returned `NormalizedString` /// should come from calling either `.split` or `.slice` on the received one. - #[text_signature = "(self, func)"] + #[pyo3(text_signature = "(self, func)")] fn split(&mut self, func: &PyAny) -> PyResult<()> { split(&mut self.pretok, func) } @@ -194,7 +194,7 @@ impl PyPreTokenizedString { /// The function used to normalize each underlying split. This function /// does not need to return anything, just calling the methods on the provided /// NormalizedString allow its modification. - #[text_signature = "(self, func)"] + #[pyo3(text_signature = "(self, func)")] fn normalize(&mut self, func: &PyAny) -> PyResult<()> { normalize(&mut self.pretok, func) } @@ -205,7 +205,7 @@ impl PyPreTokenizedString { /// func: Callable[[str], List[Token]]: /// The function used to tokenize each underlying split. This function must return /// a list of Token generated from the input str. - #[text_signature = "(self, func)"] + #[pyo3(text_signature = "(self, func)")] fn tokenize(&mut self, func: &PyAny) -> PyResult<()> { tokenize(&mut self.pretok, func) } @@ -224,7 +224,7 @@ impl PyPreTokenizedString { /// Returns: /// An Encoding #[args(type_id = "0", word_idx = "None")] - #[text_signature = "(self, type_id=0, word_idx=None)"] + #[pyo3(text_signature = "(self, type_id=0, word_idx=None)")] fn to_encoding(&self, type_id: u32, word_idx: Option) -> PyResult { to_encoding(&self.pretok, type_id, word_idx) } @@ -249,7 +249,7 @@ impl PyPreTokenizedString { offset_referential = "PyOffsetReferential(OffsetReferential::Original)", offset_type = "PyOffsetType(OffsetType::Char)" )] - #[text_signature = "(self, offset_referential=\"original\", offset_type=\"char\")"] + #[pyo3(text_signature = "(self, offset_referential=\"original\", offset_type=\"char\")")] fn get_splits( &self, offset_referential: PyOffsetReferential, @@ -259,7 +259,7 @@ impl PyPreTokenizedString { } } -#[pyclass(module = "tokenizers", name=PreTokenizedString)] +#[pyclass(module = "tokenizers", name = "PreTokenizedString")] #[derive(Clone)] pub struct PyPreTokenizedStringRefMut { inner: RefMutContainer, diff --git a/bindings/python/src/utils/regex.rs b/bindings/python/src/utils/regex.rs index 8170ffc31..9e0d4240e 100644 --- a/bindings/python/src/utils/regex.rs +++ b/bindings/python/src/utils/regex.rs @@ -3,8 +3,8 @@ use pyo3::exceptions; use pyo3::prelude::*; /// Instantiate a new Regex with the given pattern -#[pyclass(module = "tokenizers", name=Regex)] -#[text_signature = "(self, pattern)"] +#[pyclass(module = "tokenizers", name = "Regex")] +#[pyo3(text_signature = "(self, pattern)")] pub struct PyRegex { pub inner: Regex, pub pattern: String,